diff --git a/Plugins/Mineplex.ServerMonitor/pom.xml b/Plugins/Mineplex.ServerMonitor/pom.xml index d9dc9bf60..ce25de618 100644 --- a/Plugins/Mineplex.ServerMonitor/pom.xml +++ b/Plugins/Mineplex.ServerMonitor/pom.xml @@ -12,6 +12,11 @@ ServerMonitor mineplex-servermonitor + + + UTF-8 + 2.12 + @@ -28,6 +33,12 @@ org.apache.httpcomponents httpclient + + jline + jline + ${version.jline} + compile + diff --git a/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java b/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java index 086e356f2..2594699fd 100644 --- a/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java +++ b/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java @@ -16,11 +16,15 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Timer; import java.util.TimerTask; import java.util.logging.FileHandler; import java.util.logging.Logger; +import org.bukkit.craftbukkit.libs.com.google.common.base.Throwables; +import org.fusesource.jansi.AnsiConsole; + import mineplex.core.common.util.NautHashMap; import mineplex.serverdata.Region; import mineplex.serverdata.commands.RestartCommand; @@ -32,9 +36,11 @@ import mineplex.serverdata.servers.DedicatedServerSorter; import mineplex.serverdata.servers.ServerManager; import mineplex.serverdata.servers.ServerRepository; +import jline.console.ConsoleReader; + public class ServerMonitor { - private static ServerRepository _repository = null; + private static volatile ServerRepository _repository = null; private static StatusHistoryRepository _historyRepository = null; private static int _count = 0; private static HashSet _processes = new HashSet(); @@ -42,18 +48,18 @@ public class ServerMonitor private static Collection _serverStatuses = null; private static Collection _serverGroups = null; private static Map _serverGroupMap = null; - private static List _dedicatedServers = null; + private static volatile List _dedicatedServers = null; private static HashSet _deadServers = new HashSet(); private static HashSet _delayedKill = new HashSet(); private static HashSet _laggyServers = new HashSet(); - - private static SimpleDateFormat _dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); - private static Logger _logger = Logger.getLogger("ServerMonitor"); + + private static volatile SimpleDateFormat _dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); + private static volatile Logger _logger = Logger.getLogger("ServerMonitor"); private static Timer _timer = new Timer(); - + private static int _totalPlayers = 0; private static Region _region; - + private static boolean _debug = false; public static void main (String args[]) @@ -75,7 +81,7 @@ public class ServerMonitor _repository = ServerManager.getServerRepository(_region); // Fetches and connects to server repo _historyRepository = new StatusHistoryRepository(); File logFile = new File("monitor.log"); - + if (!logFile.exists()) { try @@ -87,7 +93,7 @@ public class ServerMonitor e1.printStackTrace(); } } - + try { FileHandler fileHandler = new FileHandler("monitor.log", true); @@ -99,9 +105,53 @@ public class ServerMonitor { e1.printStackTrace(); } - + + new Thread(new Runnable() + { + @Override + public void run() + { + try + { + AnsiConsole.systemInstall(); + ConsoleReader reader = new ConsoleReader(); + reader.setExpandEvents(false); + + String command; + while ((command = reader.readLine(">")) != null) + { + if (command.contains("printservers")) + { + String[] parts = command.split(" "); + String serverGroup = ""; + if (parts.length > 1) + { + serverGroup = parts[1]; + } + + printServersCommand(serverGroup); + } + else if (command.equals("help")) + { + log("Commands:"); + log("printservers (server-group) - prints all servers if no server group is" + + "supplied, or all servers of a given group if a server group is supplied."); + } + else + { + log("No command " + command + " found."); + } + } + } + catch (Exception e) + { + log(Throwables.getStackTraceAsString(e)); + } + } + }).start(); + HashMap> serverTracker = new HashMap>(); - + while (true) { try @@ -110,22 +160,22 @@ public class ServerMonitor _serverStatuses = _repository.getServerStatuses(); _serverGroups = _repository.getServerGroups(_serverStatuses); _serverGroupMap = new HashMap(); - _dedicatedServers = new ArrayList(_repository.getDedicatedServers()); - + _dedicatedServers = Collections.synchronizedList(new ArrayList<>(_repository.getDedicatedServers())); + calculateTotalPlayers(); killDeadServers(); - + double totalCPU = 0.0; double totalRAM = 0.0; double availableCPU = 0.0; double availableRAM = 0.0; - + for (DedicatedServer server : _dedicatedServers) { totalCPU += server.getAvailableCpu(); totalRAM += server.getAvailableRam(); } - + for (MinecraftServer minecraftServer : _serverStatuses) { if (!ignoreServer(minecraftServer.getGroup())) @@ -133,12 +183,12 @@ public class ServerMonitor if (minecraftServer.getMotd().contains("Finished") || (minecraftServer.getGroup().equalsIgnoreCase("UltraHardcore") && minecraftServer.getMotd().contains("Restarting") && minecraftServer.getPlayerCount() == 0)) { killServer(minecraftServer.getName(), minecraftServer.getPublicAddress(), minecraftServer.getPlayerCount(), "[KILLED] [FINISHED] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress(), true); - + handleUserServerGroup(_serverGroupMap.get(minecraftServer.getGroup())); continue; } } - + for (DedicatedServer server : _dedicatedServers) { if (_serverGroupMap.containsKey(minecraftServer.getGroup()) && minecraftServer.getPublicAddress().equalsIgnoreCase(server.getPrivateAddress())) @@ -148,11 +198,11 @@ public class ServerMonitor } } } - + if (_count % 15 == 0) - { + { _badServers.clear(); - + for (DedicatedServer serverData : _dedicatedServers) { if (isServerOffline(serverData)) @@ -161,14 +211,14 @@ public class ServerMonitor _badServers.put(serverData.getName(), true); } } - + log(_badServers.size() + " bad servers."); } - + for (Iterator iterator = _dedicatedServers.iterator(); iterator.hasNext();) - { + { DedicatedServer serverData = iterator.next(); - + if (_badServers.containsKey(serverData.getName())) iterator.remove(); else @@ -177,12 +227,12 @@ public class ServerMonitor availableRAM += serverData.getAvailableRam(); } } - + double usedCpuPercent = Math.round((1 - availableCPU / totalCPU) * 10000.0) / 100.0; double usedRamPercent = Math.round((1 - availableRAM / totalRAM) * 10000.0) / 100.0; - + log("Using " + usedCpuPercent + "% of available CPU (" + availableCPU + " Free) and " + usedRamPercent + "% of available RAM (" + availableRAM / 1000 + "GB Free)"); - + try { _historyRepository.saveDedicatedServerStats(_dedicatedServers); @@ -196,19 +246,19 @@ public class ServerMonitor { ex.printStackTrace(); } - + if (_count == 0) { for (Iterator groupStatusIterator = _serverGroups.iterator(); groupStatusIterator.hasNext();) { ServerGroup groupStatus = groupStatusIterator.next(); - + if (groupStatus.getServerType().equalsIgnoreCase("Player")) { _repository.removeServerGroup(groupStatus); _serverGroupMap.remove(groupStatus.getName()); groupStatusIterator.remove(); - + System.out.println("Removed MPS : " + groupStatus.getName()); } if (groupStatus.getServerType().equalsIgnoreCase("Community")) @@ -216,19 +266,19 @@ public class ServerMonitor _repository.removeServerGroup(groupStatus); _serverGroupMap.remove(groupStatus.getName()); groupStatusIterator.remove(); - + System.out.println("Removed MCS : " + groupStatus.getName()); } } } - + for (ServerGroup groupStatus : _serverGroups) { if (ignoreServer(groupStatus.getName())) continue; - + NautHashMap serverMap = new NautHashMap(); - + for (Iterator serverIterator = groupStatus.getServers().iterator(); serverIterator.hasNext();) { try @@ -236,7 +286,7 @@ public class ServerMonitor MinecraftServer server = serverIterator.next(); String[] nameArgs = server.getName().split("-"); int serverNum = Integer.parseInt(nameArgs[nameArgs.length - 1]); - + if (serverMap.containsKey(serverNum)) { killServer(server.getName(), server.getPublicAddress(), server.getPlayerCount(), "[KILLED] [DUPLICATE] " + server.getName() + ":" + server.getPublicAddress(), true); @@ -253,19 +303,19 @@ public class ServerMonitor } } } - + HashSet onlineServers = new HashSet(); HashSet laggyServers = new HashSet(); laggyServers.addAll(_laggyServers); _laggyServers.clear(); - + for (MinecraftServer minecraftServer : _serverStatuses) { if (ignoreServer(minecraftServer.getGroup())) continue; - + onlineServers.add(minecraftServer.getName()); - + if (minecraftServer.getTps() <= 17) { if (minecraftServer.getTps() <= 10) @@ -285,27 +335,27 @@ public class ServerMonitor log("[Performance] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress() + "] Running poorly at " + minecraftServer.getTps() + " TPS"); } } - + for (Iterator>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();) { Entry> entry = iterator.next(); - + if (onlineServers.contains(entry.getKey())) iterator.remove(); else if (System.currentTimeMillis() - entry.getValue().getValue() > 35000) - { + { String serverName = entry.getKey(); String serverAddress = entry.getValue().getKey(); killServer(serverName, serverAddress, 0, "[KILLED] [SLOW-STARTUP] " + serverName + ":" + serverAddress, true); iterator.remove(); } } - + for (ServerGroup serverGroup : _serverGroups) { if (ignoreServer(serverGroup.getName())) continue; - + try { handleGroupChanges(serverTracker, serverGroup, false); @@ -316,9 +366,9 @@ public class ServerMonitor log("Can't handle group changes for " + serverGroup.getName()); } } - + int processWaits = 0; - + while (_processes.size() > 0) { try @@ -326,16 +376,16 @@ public class ServerMonitor for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { ProcessRunner pr = iterator.next(); - - try + + try { pr.join(100); - } - catch (InterruptedException e) + } + catch (InterruptedException e) { e.printStackTrace(); } - + if (pr.isDone()) iterator.remove(); } @@ -344,48 +394,48 @@ public class ServerMonitor { ex.printStackTrace(); } - + if (_processes.size() > 0) { try { log("Sleeping while processes run..."); Thread.sleep(6000); - } + } catch (InterruptedException e) { e.printStackTrace(); } } - + if (processWaits >= 5) { log("Killing stale processes."); - + for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { iterator.next().abort(); iterator.remove(); } - + _processes.clear(); } - + processWaits++; } - + processWaits = 0; - + try { log("Natural sleep."); Thread.sleep(10000); - } + } catch (InterruptedException e) { e.printStackTrace(); } - + _count++; } catch (Exception ex) @@ -408,40 +458,41 @@ public class ServerMonitor { HashSet deadServers = new HashSet(); deadServers.addAll(_deadServers); - + _deadServers.clear(); for (final MinecraftServer deadServer : _repository.getDeadServers()) { if (deadServer.getUptime() <= 10 || ignoreServer(deadServer.getGroup())) continue; - + try { if (_count == 0 || deadServers.contains(deadServer.getName())) { if (_count != 0) copyServerLog(deadServer); - + killServer(deadServer.getName(), deadServer.getPublicAddress(), deadServer.getPlayerCount(), "[KILLED] [DEAD] " + deadServer.getName() + ":" + deadServer.getPublicAddress(), true); - + handleUserServerGroup(_serverGroupMap.get(deadServer.getGroup())); } else if (!_delayedKill.contains(deadServer.getName())) - { + { startTimingReport(deadServer); - + _timer.schedule(new TimerTask() { - public void run() + @Override + public void run() { _deadServers.add(deadServer.getName()); _delayedKill.remove(deadServer.getName()); - + stopTimingReport(deadServer); log("[IMPENDING DEATH] : " + deadServer.getName() + ":" + deadServer.getPublicAddress()); } }, 20 * 1000); - + _delayedKill.add(deadServer.getName()); } } @@ -463,7 +514,7 @@ public class ServerMonitor System.out.println("Removed ServerGroup : " + serverGroup.getName()); } } - + private static void calculateTotalPlayers() { for (ServerGroup serverGroup : _serverGroups) @@ -471,7 +522,7 @@ public class ServerMonitor _serverGroupMap.put(serverGroup.getName(), serverGroup); _totalPlayers += serverGroup.getPlayerCount(); } - + log("Total Players : " + _totalPlayers); } @@ -490,13 +541,13 @@ public class ServerMonitor if (serverGroup.getName().equalsIgnoreCase("Lobby")) { if (_region == Region.EU) - { + { requiredTotal = 10; requiredJoinable = 10; } - + int availableSlots = serverGroup.getMaxPlayerCount() - serverGroup.getPlayerCount(); - + if (availableSlots < 1000) { serversToAdd = Math.max(1, (1000 - availableSlots) / serverGroup.getMaxPlayers()); @@ -509,12 +560,12 @@ public class ServerMonitor { serversToRestart = Math.min(joinableServers - requiredJoinable, joinableServers - requiredTotal); serversToRestart = Math.min(serversToRestart, (availableSlots - 1000) / 80); - + if (serversToRestart <= 5) serversToRestart = 0; } - - + + } else if (serverGroup.getName().equalsIgnoreCase("Halloween")) { @@ -533,10 +584,10 @@ public class ServerMonitor else if (serverGroup.getName().equalsIgnoreCase("UltraHardcore")) { int maxUHC = Math.max(1, _totalPlayers / 6000); - + if (serversToAdd > 0) serversToAdd = maxUHC - joinableServers; - + if (joinableServers > maxUHC) serversToKill = maxUHC - joinableServers; } @@ -544,7 +595,7 @@ public class ServerMonitor { return; } - + // KILL, CLEAN, THEN ADD while (serversToKill > 0) { @@ -553,16 +604,16 @@ public class ServerMonitor killServer(emptyServer, "[KILLED] [EXCESS] " + emptyServer.getName() + ":" + emptyServer.getPublicAddress()); serversToKill--; } - + while (serversToAdd > 0) { serverNum = serverGroup.generateUniqueId(serverNum + 1); - + while (_deadServers.contains(serverGroup.getPrefix() + "-" + serverNum)) { serverNum = serverGroup.generateUniqueId(serverNum + 1); } - + Collections.sort(_dedicatedServers, new DedicatedServerSorter()); DedicatedServer bestServer = getBestDedicatedServer(_dedicatedServers, serverGroup); @@ -579,14 +630,14 @@ public class ServerMonitor startServer(bestServer, serverGroup, serverNum, free); serverTracker.put(serverGroup.getPrefix() + "-" + serverNum, new AbstractMap.SimpleEntry(bestServer.getPublicAddress(), System.currentTimeMillis())); } - + serversToAdd--; } - + List servers = new ArrayList(); servers.addAll(serverGroup.getServers()); Collections.sort(servers, new ServerSorter()); - + while (serversToRestart > 0) { MinecraftServer server = servers.get(servers.size() - serversToRestart); @@ -600,26 +651,27 @@ public class ServerMonitor { if (_debug) return; - + String cmd = "/home/mineplex/easyRemoteKillServer.sh"; - + ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverName}); - pr.start(new GenericRunnable() + pr.start(new GenericRunnable() { - public void run(Boolean error) + @Override + public void run(Boolean error) { MinecraftServer server = null; - + if (!error) { server = _repository.getServerStatus(serverName); - + if (server != null) { _repository.removeServerStatus(server); } } - + if (announce) { if (error) @@ -629,17 +681,17 @@ public class ServerMonitor } } }); - - try + + try { pr.join(50); - } - catch (InterruptedException e1) + } + catch (InterruptedException e1) { e1.printStackTrace(); } - - + + if (!pr.isDone()) _processes.add(pr); } @@ -647,31 +699,31 @@ public class ServerMonitor private static boolean isServerOffline(DedicatedServer serverData) { boolean success = false; - + if (_debug) return false; - + Process process = null; String cmd = "/home/mineplex/isServerOnline.sh"; - + ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.getPublicAddress()}); - + try { process = processBuilder.start(); process.waitFor(); - - BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); - String line = reader.readLine(); - - while(line != null) + + BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = reader.readLine(); + + while(line != null) { success = line.equals("Success"); - - line=reader.readLine(); - } - } - catch (Exception e1) + + line=reader.readLine(); + } + } + catch (Exception e1) { e1.printStackTrace(); } @@ -686,10 +738,10 @@ public class ServerMonitor private static DedicatedServer getBestDedicatedServer(Collection dedicatedServers, ServerGroup serverGroup) { DedicatedServer bestServer = null; - + for (DedicatedServer serverData : dedicatedServers) { - if (serverData.getAvailableRam() > serverGroup.getRequiredRam() + if (serverData.getAvailableRam() > serverGroup.getRequiredRam() && serverData.getAvailableCpu() > serverGroup.getRequiredCpu()) { if (bestServer == null || serverData.getServerCount(serverGroup) < bestServer.getServerCount(serverGroup)) @@ -698,7 +750,7 @@ public class ServerMonitor } } } - + return bestServer; } @@ -706,18 +758,19 @@ public class ServerMonitor { killServer(serverToKill.getName(), serverToKill.getPublicAddress(), serverToKill.getPlayerCount(), message, true); } - + private static void startTimingReport(final MinecraftServer server) { if (_debug) return; - + String cmd = "/home/mineplex/remoteStartTiming.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, server.getPublicAddress(), server.getName() }); - pr.start(new GenericRunnable() + pr.start(new GenericRunnable() { - public void run(Boolean error) + @Override + public void run(Boolean error) { if (error) log("[TIMING START] Errored " + server.getName() + "(" + server.getPublicAddress() + ")"); @@ -726,31 +779,32 @@ public class ServerMonitor } }); - - try + + try { pr.join(100); - } - catch (InterruptedException e1) + } + catch (InterruptedException e1) { e1.printStackTrace(); } - + if (!pr.isDone()) _processes.add(pr); } - + private static void stopTimingReport(final MinecraftServer server) { if (_debug) return; - + String cmd = "/home/mineplex/remoteStopTiming.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, server.getPublicAddress(), server.getName() }); - pr.start(new GenericRunnable() + pr.start(new GenericRunnable() { - public void run(Boolean error) + @Override + public void run(Boolean error) { if (error) log("[TIMING PASTE] Errored " + server.getName() + "(" + server.getPublicAddress() + ")"); @@ -759,31 +813,32 @@ public class ServerMonitor } }); - - try + + try { pr.join(100); - } - catch (InterruptedException e1) + } + catch (InterruptedException e1) { e1.printStackTrace(); } - + if (!pr.isDone()) _processes.add(pr); } - + private static void copyServerLog(final MinecraftServer server) { if (_debug) return; - + String cmd = "/home/mineplex/easyRemoteCopyLog.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, server.getPublicAddress(), server.getName() }); - pr.start(new GenericRunnable() + pr.start(new GenericRunnable() { - public void run(Boolean error) + @Override + public void run(Boolean error) { if (error) log("[COPY LOG] Errored " + server.getName() + "(" + server.getPublicAddress() + ")"); @@ -792,34 +847,35 @@ public class ServerMonitor } }); - - try + + try { pr.join(100); - } - catch (InterruptedException e1) + } + catch (InterruptedException e1) { e1.printStackTrace(); } - + if (!pr.isDone()) _processes.add(pr); } - + private static void startServer(final DedicatedServer serverSpace, final ServerGroup serverGroup, final int serverNum, final boolean free) { if (_debug) return; - + String cmd = "/home/mineplex/easyRemoteStartServerCustom.sh"; final String groupPrefix = serverGroup.getPrefix(); final String serverName = serverSpace.getName(); final String serverAddress = serverSpace.getPublicAddress(); ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverSpace.getPrivateAddress(), (serverGroup.getPortSection() + serverNum) + "", serverGroup.getRequiredRam() + "", serverGroup.getWorldZip(), serverGroup.getPlugin(), serverGroup.getConfigPath(), serverGroup.getName(), serverGroup.getPrefix() + "-" + serverNum, serverSpace.isUsRegion() ? "true" : "false", serverGroup.getAddNoCheat() + "", serverGroup.getAddWorldEdit() + "" }); - pr.start(new GenericRunnable() + pr.start(new GenericRunnable() { - public void run(Boolean error) + @Override + public void run(Boolean error) { if (error) log("[" + serverName + ":" + serverAddress + " Free Resources; CPU " + serverSpace.getAvailableCpu() + " RAM " + serverSpace.getAvailableRam() + "MB] Errored " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")"); @@ -828,36 +884,68 @@ public class ServerMonitor } }); - - try + + try { pr.join(100); - } - catch (InterruptedException e1) + } + catch (InterruptedException e1) { e1.printStackTrace(); } - + serverSpace.incrementServerCount(serverGroup); - + if (!pr.isDone()) _processes.add(pr); } - + + private static void printServersCommand(String serverGroup) + { + // grab servers + Collection servers = _repository.getServerStatuses(); + + // remove entries not within the target server group, if one is supplied + if (!serverGroup.isEmpty()) + { + servers.removeIf(s -> !s.getGroup().equalsIgnoreCase(serverGroup)); + } + + List serverList = new ArrayList<>(servers); + Collections.sort(serverList, + (a, b) -> Integer.compare(b.getPlayerCount(), a.getPlayerCount())); + + serverList.forEach(server -> + { + Optional opt = getDediStatusFromAddress(server.getPublicAddress()); + String dediName = opt.map(dedi -> dedi.getName()).orElse("???"); + + String fmt = "%s: %s (%s players online), open slots: %s, free ram: %sMB"; + log(String.format(fmt, server.getName(), dediName, server.getPlayerCount(), + server.getMaxPlayerCount() - server.getPlayerCount(), server.getRam())); + }); + } + + private static Optional getDediStatusFromAddress(String address) + { + return _dedicatedServers.stream().filter(dedi -> dedi.getPrivateAddress().equals(address)) + .findFirst(); + } + private static boolean ignoreServer(String serverGroupName) { return serverGroupName.equalsIgnoreCase("Testing") || serverGroupName.equalsIgnoreCase("Clans"); } - + private static void log(String message) { log(message, false); } - + private static void log(String message, boolean fileOnly) { _logger.info("[" + _dateFormat.format(new Date()) + "] " + message); - + if (!fileOnly) System.out.println("[" + _dateFormat.format(new Date()) + "] " + message); }