From 482e603af656af0ba3ce962997bf8274779b8fda Mon Sep 17 00:00:00 2001 From: Dan Mulloy Date: Thu, 30 Nov 2017 13:44:46 -0500 Subject: [PATCH] Only hold community data in memory when a member is online This should save a ton of memory and processing power, but needs to be tested --- .../core/communities/CommunityManager.java | 93 ++++++++++----- .../core/communities/CommunityMemberData.java | 32 +++-- .../gui/overview/CommunityOverviewPage.java | 2 +- .../storage/CommunityRepository.java | 109 +++++++++++++++++- .../src/mineplex/core/monitor/LagMeter.java | 23 +++- 5 files changed, 214 insertions(+), 45 deletions(-) diff --git a/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityManager.java b/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityManager.java index d5ddb7531..68bd4bf44 100644 --- a/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityManager.java +++ b/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityManager.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.bukkit.Bukkit; import org.bukkit.entity.Player; @@ -19,10 +20,11 @@ import org.bukkit.event.EventHandler; import org.bukkit.event.EventPriority; import org.bukkit.event.player.AsyncPlayerChatEvent; import org.bukkit.event.player.PlayerJoinEvent; -import org.bukkit.plugin.java.JavaPlugin; +import org.bukkit.event.player.PlayerQuitEvent; import mineplex.core.Managers; import mineplex.core.MiniDbClientPlugin; +import mineplex.core.ReflectivelyCreateMiniPlugin; import mineplex.core.account.CoreClientManager; import mineplex.core.account.ILoginProcessor; import mineplex.core.account.permissions.Permission; @@ -69,8 +71,8 @@ import mineplex.serverdata.data.DataRepository; import mineplex.serverdata.data.PlayerStatus; import mineplex.serverdata.redis.RedisDataRepository; import mineplex.serverdata.servers.ServerManager; -import mineplex.serverdata.servers.ServerRepository; +@ReflectivelyCreateMiniPlugin public class CommunityManager extends MiniDbClientPlugin { public enum Perm implements Permission @@ -104,33 +106,33 @@ public class CommunityManager extends MiniDbClientPlugin public final List BrowserIds = new LinkedList<>(); private final List _creating = new ArrayList<>(); - public final DataRepository StatusRepository; + // private final DataRepository StatusRepository; - private ServerRepository _serverRepo; + // private ServerRepository _serverRepo; private boolean _us; private final Set dirty = Collections.newSetFromMap(new ConcurrentHashMap<>()); // Communities with redis updates private int _updateCycleCount; // The number of update cycles since we've updated all communities private volatile boolean _cycling = false; + private CoreClientManager _clientManager; + @SuppressWarnings("deprecation") - public CommunityManager(JavaPlugin plugin, CoreClientManager clientManager) + public CommunityManager() { - super("Communities", plugin, clientManager); + super("Communities"); + + DataRepository statusRepo = new RedisDataRepository<>(ServerManager.getMasterConnection(), + ServerManager.getSlaveConnection(), Region.currentRegion(), PlayerStatus.class, "playerStatus"); - StatusRepository = new RedisDataRepository(ServerManager.getMasterConnection(), ServerManager.getSlaveConnection(), - Region.currentRegion(), PlayerStatus.class, "playerStatus"); - - _us = plugin.getConfig().getBoolean("serverstatus.us"); - - Region region = _us ? Region.US : Region.EU; - _serverRepo = ServerManager.getServerRepository(region); - - _repo = new CommunityRepository(plugin, StatusRepository, _us); + _us = _plugin.getConfig().getBoolean("serverstatus.us"); + + _repo = new CommunityRepository(_plugin, statusRepo, _us); _loadedCommunities = new ConcurrentHashMap<>(); - - clientManager.addStoredProcedureLoginProcessor(new ILoginProcessor() + + _clientManager = require(CoreClientManager.class); + _clientManager.addStoredProcedureLoginProcessor(new ILoginProcessor() { @Override public String getName() @@ -158,7 +160,7 @@ public class CommunityManager extends MiniDbClientPlugin } }); - Bukkit.getScheduler().scheduleAsyncRepeatingTask(plugin, () -> + Bukkit.getScheduler().scheduleAsyncRepeatingTask(_plugin, () -> { _updateCycleCount++; @@ -174,23 +176,20 @@ public class CommunityManager extends MiniDbClientPlugin _updateCycleCount = 0; dirty.clear(); - _loadedCommunities.values().forEach(communities::add); + communities.addAll(_loadedCommunities.values()); } else { - dirty.forEach(communities::add); + communities.addAll(dirty); dirty.clear(); } _repo.updateMembersAndJoinRequests(communities); }, 0L, 20 * UPDATE_CYCLE_SECONDS); - Bukkit.getScheduler().scheduleSyncRepeatingTask(plugin, () -> - { - cycleBrowser(); - }, 0L, 20 * 30); + Bukkit.getScheduler().scheduleSyncRepeatingTask(_plugin, this::cycleBrowser, 0L, 20 * 30); - _repo.loadCommunities(_loadedCommunities); + // _repo.loadCommunities(_loadedCommunities); addCommand(new CommunityCommand(this)); @@ -262,7 +261,12 @@ public class CommunityManager extends MiniDbClientPlugin }); }); } - + + public int getCount() + { + return _loadedCommunities.size(); + } + public Community getLoadedCommunity(Integer id) { return _loadedCommunities.get(id); @@ -827,6 +831,43 @@ public class CommunityManager extends MiniDbClientPlugin } } + @EventHandler(priority = EventPriority.MONITOR) + public void onPlayerJoin(PlayerJoinEvent event) + { + // Load their communities if it hasn't already been done + Player player = event.getPlayer(); + Set communityIds = Get(player).getCommunityIds(); + + final List load = communityIds.stream().filter(id -> getLoadedCommunity(id) == null).collect(Collectors.toList()); + if (load.isEmpty()) return; + + final int accountId = _clientManager.getAccountId(player); + + runAsync(() -> _repo.loadCommunities(_loadedCommunities, load, accountId)); + } + + @EventHandler(priority = EventPriority.MONITOR) + public void onPlayerQuit(PlayerQuitEvent event) + { + // Remove their communities from memory if they're the last player + Player player = event.getPlayer(); + List communities = Get(player).getCommunities(); + com: for (Community community : communities) + { + for (UUID uuid : community.getMembers().keySet()) + { + // See if there's anyone else online besides our quitting player + if (!player.getUniqueId().equals(uuid) && Bukkit.getPlayer(uuid) != null) + { + break com; + } + } + + // Unload this community from memory + _loadedCommunities.remove(community.getId()); + } + } + @Override protected CommunityMemberData addPlayer(UUID uuid) { diff --git a/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityMemberData.java b/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityMemberData.java index 72c44fe00..aa1fe39b8 100644 --- a/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityMemberData.java +++ b/Plugins/Mineplex.Core/src/mineplex/core/communities/CommunityMemberData.java @@ -1,10 +1,14 @@ package mineplex.core.communities; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.ImmutableSet; + import mineplex.core.Managers; public class CommunityMemberData @@ -39,19 +43,25 @@ public class CommunityMemberData { _chattingTo = community.getId(); } - - public Community[] getCommunities() + + Set getCommunityIds() { - Community[] communities = new Community[_communities.size()]; - int index = 0; - for (Integer comId : _communities.keySet()) - { - communities[index] = Managers.get(CommunityManager.class).getLoadedCommunity(comId); - index++; - } - return communities; + return _communities.keySet(); } - + + public List getCommunities() + { + List ret = new ArrayList<>(_communities.size()); + for (int id : _communities.keySet()) + { + Community community = Managers.get(CommunityManager.class).getLoadedCommunity(id); + if (community != null) + ret.add(community); + } + + return ret; + } + public boolean isMemberOf(Community community) { return _communities.containsKey(community.getId()); diff --git a/Plugins/Mineplex.Core/src/mineplex/core/communities/gui/overview/CommunityOverviewPage.java b/Plugins/Mineplex.Core/src/mineplex/core/communities/gui/overview/CommunityOverviewPage.java index 342afa5bd..e8a9da5cc 100644 --- a/Plugins/Mineplex.Core/src/mineplex/core/communities/gui/overview/CommunityOverviewPage.java +++ b/Plugins/Mineplex.Core/src/mineplex/core/communities/gui/overview/CommunityOverviewPage.java @@ -90,7 +90,7 @@ public class CommunityOverviewPage extends CommunitiesGUIPage Inv.setItem(clear, null); } } - List coms = Arrays.asList(getCommunityManager().Get(Viewer).getCommunities()); + List coms = getCommunityManager().Get(Viewer).getCommunities(); coms.sort((c1, c2) -> { if (c1.getMembers().get(Viewer.getUniqueId()).Role == c2.getMembers().get(Viewer.getUniqueId()).Role) diff --git a/Plugins/Mineplex.Core/src/mineplex/core/communities/storage/CommunityRepository.java b/Plugins/Mineplex.Core/src/mineplex/core/communities/storage/CommunityRepository.java index 4d9d45992..4a01d0c7f 100644 --- a/Plugins/Mineplex.Core/src/mineplex/core/communities/storage/CommunityRepository.java +++ b/Plugins/Mineplex.Core/src/mineplex/core/communities/storage/CommunityRepository.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,13 +30,16 @@ import mineplex.serverdata.database.column.ColumnVarChar; public class CommunityRepository extends RepositoryBase { + private static final String GET_COMMUNITIES_BY_ID = "SELECT * FROM communities"; + private static final String GET_COMMUNITY_MEMBERS = "SELECT cm.communityId, cm.accountId, cm.communityRole, ac.name, ac.uuid, ac.lastLogin, cm.readingChat FROM communityMembers cm INNER JOIN accounts ac ON ac.id=cm.accountId"; + private static final String GET_COMMUNITY_JOIN_REQUESTS = "SELECT cjr.communityId, cjr.accountId, ac.name, ac.uuid FROM communityJoinRequests cjr INNER JOIN accounts ac ON ac.id=cjr.accountId WHERE cjr.accountId=?;"; + private static final String GET_COMMUNITY_SETTINGS = "SELECT communityId, settingId, settingValue FROM communitySettings"; + + // Old queries private static final String GET_ALL_COMMUNITIES = "SELECT * FROM communities WHERE region=?;"; private static final String GET_COMMUNITY_BY_ID = "SELECT * FROM communities WHERE id=?;"; private static final String GET_COMMUNITY_BY_NAME = "SELECT * FROM communities WHERE name=? AND region=?;"; - private static final String GET_COMMUNITY_MEMBERS = "SELECT cm.communityId, cm.accountId, cm.communityRole, ac.name, ac.uuid, ac.lastLogin, cm.readingChat FROM communityMembers cm INNER JOIN accounts ac ON ac.id=cm.accountId;"; - private static final String GET_COMMUNITY_JOIN_REQUESTS = "SELECT cjr.communityId, cjr.accountId, ac.name, ac.uuid FROM communityJoinRequests cjr INNER JOIN accounts ac ON ac.id=cjr.accountId;"; - private static final String GET_COMMUNITY_SETTINGS = "SELECT communityId, settingId, settingValue FROM communitySettings;"; - + private static final String REMOVE_FROM_COMMUNITY = "DELETE FROM communityMembers WHERE accountId=? AND communityId=?;"; private static final String UPDATE_COMMUNITY_ROLE = "UPDATE communityMembers SET communityRole=? WHERE accountId=? AND communityId=?;"; private static final String ADD_TO_COMMUNITY = "INSERT INTO communityMembers (accountId, communityId, communityRole, readingChat) VALUES (?, ?, ?, true);"; @@ -61,7 +65,100 @@ public class CommunityRepository extends RepositoryBase _us = us; } - public void loadCommunities(final Map communityMap) + // TODO this could probably be further optimized, either way it needs to be tested + public void loadCommunities(final Map store, final List load, final int accountId) + { + try (Connection connection = getConnection()) + { + // Only load the info for communities with the given IDs + StringBuilder builder = new StringBuilder(); + builder.append(" WHERE id IN ("); + + for (int index = 0; index < load.size(); index++) + { + if (index != 0) + builder.append(", "); + builder.append("?"); + } + + builder.append(");"); + + String inClause = builder.toString(); + ColumnInt[] idColumns = load.stream().map(i -> new ColumnInt("id", i)).toArray(x -> new ColumnInt[0]); + + executeQuery(connection, GET_COMMUNITIES_BY_ID + inClause, resultSet -> + { + final int id = resultSet.getInt("id"); + final String cName = resultSet.getString("name"); + final Community community = new Community(id, cName); + + store.put(id, community); + }, idColumns); + + executeQuery(connection, GET_COMMUNITY_MEMBERS + inClause, memberSet -> + { + while (memberSet.next()) + { + final int communityId = memberSet.getInt("communityId"); + final int accountId1 = memberSet.getInt("accountId"); + final String name = memberSet.getString("name"); + final UUID uuid = UUID.fromString(memberSet.getString("uuid")); + final CommunityRole role = CommunityRole.parseRole(memberSet.getString("communityRole")); + final long lastLogin = memberSet.getTimestamp("lastLogin").getTime(); + boolean readingChat = memberSet.getBoolean("readingChat"); + + CommunityMemberInfo info = new CommunityMemberInfo(name, uuid, accountId1, role, lastLogin); + info.ReadingChat = readingChat; + + Community community = store.get(communityId); + if (community != null) + { + community.getMembers().put(info.UUID, info); + } + } + }, idColumns); + + executeQuery(connection, GET_COMMUNITY_JOIN_REQUESTS, requestSet -> + { + while (requestSet.next()) + { + final int communityId = requestSet.getInt("communityId"); + // final int accountId = requestSet.getInt("accountId"); + final UUID uuid = UUID.fromString(requestSet.getString("uuid")); + final String name = requestSet.getString("name"); + + Community community = store.get(communityId); + if (community != null) + { + community.getJoinRequests().put(uuid, new CommunityJoinRequestInfo(name, uuid, accountId)); + } + } + }, new ColumnInt("cjr.accountId", accountId)); + + executeQuery(connection, GET_COMMUNITY_SETTINGS + inClause, settingSet -> + { + while (settingSet.next()) + { + final int communityId = settingSet.getInt("communityId"); + final int settingId = settingSet.getInt("settingId"); + final String value = settingSet.getString("settingValue"); + + Community community = store.get(communityId); + CommunitySetting setting = CommunitySetting.getSetting(settingId); + if (community != null && setting != null) + { + setting.parseValueInto(value, community); + } + } + }, idColumns); + } catch (SQLException ex) + { + System.err.println("Encountered an SQL exception loading communities " + load); + ex.printStackTrace(); + } + } + + /* public void loadCommunities(final Map communityMap) { try (Connection connection = getConnection()) { @@ -142,7 +239,7 @@ public class CommunityRepository extends RepositoryBase { e.printStackTrace(); } - } + } */ public void updateMembersAndJoinRequests(List communities) { diff --git a/Plugins/Mineplex.Core/src/mineplex/core/monitor/LagMeter.java b/Plugins/Mineplex.Core/src/mineplex/core/monitor/LagMeter.java index 3c6337a7b..41d9ceb15 100644 --- a/Plugins/Mineplex.Core/src/mineplex/core/monitor/LagMeter.java +++ b/Plugins/Mineplex.Core/src/mineplex/core/monitor/LagMeter.java @@ -11,12 +11,14 @@ import org.bukkit.event.player.PlayerCommandPreprocessEvent; import org.bukkit.event.player.PlayerQuitEvent; import org.bukkit.plugin.java.JavaPlugin; +import mineplex.core.Managers; import mineplex.core.MiniPlugin; import mineplex.core.account.CoreClientManager; import mineplex.core.account.permissions.Permission; import mineplex.core.account.permissions.PermissionGroup; import mineplex.core.common.util.C; import mineplex.core.common.util.F; +import mineplex.core.communities.CommunityManager; import mineplex.core.updater.UpdateType; import mineplex.core.updater.event.UpdateEvent; @@ -30,6 +32,8 @@ public class LagMeter extends MiniPlugin } private CoreClientManager _clientManager; + private CommunityManager _communities; + private long _lastRun = -1; private int _count; private double _ticksPerSecond; @@ -166,6 +170,23 @@ public class LagMeter extends MiniPlugin player.sendMessage(F.main(getName(), ChatColor.YELLOW + "MEM")); player.sendMessage(F.main(getName(), ChatColor.GRAY + "Free-------" + ChatColor.YELLOW + (Runtime.getRuntime().freeMemory() / 1048576) + "MB")); player.sendMessage(F.main(getName(), ChatColor.GRAY + "Max--------" + ChatColor.YELLOW + (Runtime.getRuntime().maxMemory() / 1048576)) + "MB"); - player.sendMessage(F.main(getName(), ChatColor.YELLOW + String.valueOf(player.getWorld().getLoadedChunks().length) + ChatColor.GRAY + " chunks loaded")); + + // Statistics for Dan; should be temporary + if (_clientManager.Get(player).getPrimaryGroup().inheritsFrom(PermissionGroup.DEV)) + { + player.sendMessage(" "); + player.sendMessage(F.main(getName(), ChatColor.GRAY + "Dev Stats -----")); + + player.sendMessage(F.main(getName(), + ChatColor.YELLOW + String.valueOf(player.getWorld().getLoadedChunks().length) + ChatColor.GRAY + + " chunks loaded")); + + if (_communities == null) + _communities = Managers.get(CommunityManager.class); + + player.sendMessage(F.main(getName(), + ChatColor.YELLOW + String.valueOf(_communities.getCount()) + ChatColor.GRAY + + " communities loaded")); + } } } \ No newline at end of file