package mineplex.servermonitor; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.text.SimpleDateFormat; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.FileHandler; import java.util.logging.Logger; import mineplex.core.common.util.NautHashMap; import mineplex.serverdata.DedicatedServer; import mineplex.serverdata.DedicatedServerSorter; import mineplex.serverdata.MinecraftServer; import mineplex.serverdata.Region; import mineplex.serverdata.ServerGroup; import mineplex.serverdata.ServerManager; import mineplex.serverdata.ServerRepository; public class ServerMonitor { private static ServerRepository _repository = null; private static int _count = 0; private static HashSet _processes = new HashSet(); private static HashMap _badServers = new HashMap(); private static Collection _serverStatuses = null; private static Collection _serverGroups = null; private static Map _serverGroupMap = null; private static List _dedicatedServers = null; private static SimpleDateFormat _dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); private static Logger _logger = Logger.getLogger("ServerMonitor"); private static int _totalPlayers = 0; private static Region _region; private static boolean _debug = false; public static void main (String args[]) { _region = !new File("eu.dat").exists() ? Region.US : Region.EU; _repository = ServerManager.getServerRepository(_region); // Fetches and connects to server repo File logFile = new File("monitor.log"); if (!logFile.exists()) { try { logFile.createNewFile(); } catch (IOException e1) { e1.printStackTrace(); } } try { FileHandler fileHandler = new FileHandler("monitor.log", true); fileHandler.setFormatter(new CustomFormatter()); _logger.addHandler(fileHandler); _logger.setUseParentHandlers(false); } catch (SecurityException | IOException e1) { e1.printStackTrace(); } HashMap> serverTracker = new HashMap>(); while (true) { _totalPlayers = 0; _serverStatuses = _repository.getServerStatuses(); _serverGroups = _repository.getServerGroups(_serverStatuses); _serverGroupMap = new HashMap(); _dedicatedServers = new ArrayList(_repository.getDedicatedServers()); calculateTotalPlayers(); killDeadServers(); for (MinecraftServer minecraftServer : _serverStatuses) { if (minecraftServer.getMotd().contains("Finished") || (minecraftServer.getGroup().equalsIgnoreCase("UltraHardcore") && minecraftServer.getMotd().contains("Restarting") && minecraftServer.getPlayerCount() == 0)) { killServer(minecraftServer.getName(), minecraftServer.getPublicAddress(), "[KILLED] [FINISHED] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress(), true); handleUserServerGroup(_serverGroupMap.get(minecraftServer.getGroup())); continue; } for (DedicatedServer server : _dedicatedServers) { if (_serverGroupMap.containsKey(minecraftServer.getGroup()) && minecraftServer.getPublicAddress().equalsIgnoreCase(server.getPrivateAddress())) { ServerGroup serverGroup = _serverGroupMap.get(minecraftServer.getGroup()); server.incrementServerCount(serverGroup); } } } double totalCPU = 0.0; double totalRAM = 0.0; if (_count % 15 == 0) { _badServers.clear(); for (DedicatedServer serverData : _dedicatedServers) { if (isServerOffline(serverData)) { log("------=[OFFLINE]=------=[" + serverData.getName() + ":" + serverData.getPublicAddress() + "]=------=[OFFLINE]=------"); _badServers.put(serverData.getName(), true); } else { totalCPU += serverData.getAvailableCpu(); totalRAM += serverData.getAvailableRam(); } } log(_badServers.size() + " bad servers."); } for (Iterator iterator = _dedicatedServers.iterator(); iterator.hasNext();) { DedicatedServer serverData = iterator.next(); if (_badServers.containsKey(serverData.getName())) iterator.remove(); } for (ServerGroup groupStatus : _serverGroups) { NautHashMap serverMap = new NautHashMap(); for (Iterator serverIterator = groupStatus.getServers().iterator(); serverIterator.hasNext();) { MinecraftServer server = serverIterator.next(); int serverNum = Integer.parseInt(server.getName().split("-")[1]); if (serverMap.containsKey(serverNum)) { killServer(server.getName(), server.getPublicAddress(), "[KILLED] [DUPLICATE] " + server.getName() + ":" + server.getPublicAddress(), true); serverIterator.remove(); } else { serverMap.put(serverNum, server); } } /* if (groupStatus.getHost() == null || groupStatus.getHost().isEmpty()) { int serverCount = groupStatus.getServers().size(); log(groupStatus.getName() + " : " + groupStatus.getPlayerCount() + " players on " + serverCount + " servers " + String.format("%.2f", ((double)serverCount * (double)groupStatus.getRequiredCpu() / totalCPU)) + "% CPU," + String.format("%.2f", ((double)serverCount * (double)groupStatus.getRequiredRam() / totalRAM)) + "% RAM", true); } */ } HashSet onlineServers = new HashSet(); for (MinecraftServer minecraftServer : _serverStatuses) { onlineServers.add(minecraftServer.getName()); if (minecraftServer.getTps() <= 17) log("[Performance] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress() + "] Running poorly at " + minecraftServer.getTps() + " TPS"); } for (Iterator>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();) { Entry> entry = iterator.next(); if (onlineServers.contains(entry.getKey())) iterator.remove(); else if (System.currentTimeMillis() - entry.getValue().getValue() > 30000) { String serverName = entry.getKey(); String serverAddress = entry.getValue().getKey(); killServer(serverName, serverAddress, "[KILLED] [SLOW-STARTUP] " + serverName + ":" + serverAddress, true); iterator.remove(); } } for (ServerGroup serverGroup : _serverGroups) { if (serverGroup.getName().equals("Testing")) continue; handleGroupChanges(serverTracker, serverGroup, false); } int processWaits = 0; while (_processes.size() > 0) { for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { ProcessRunner pr = iterator.next(); try { pr.join(100); } catch (InterruptedException e) { e.printStackTrace(); } if (pr.isDone()) iterator.remove(); } if (_processes.size() > 0) { try { log("Sleeping while processes run..."); Thread.sleep(6000); } catch (InterruptedException e) { e.printStackTrace(); } } if (processWaits >= 10) { log("Killing stale processes."); for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { iterator.next().abort(); iterator.remove(); } } processWaits++; } processWaits = 0; try { log("Natural sleep."); Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); } _count++; } } private static void killDeadServers() { for (MinecraftServer deadServer : _repository.getDeadServers()) { killServer(deadServer.getName(), deadServer.getPublicAddress(), "[KILLED] [DEAD] " + deadServer.getName() + ":" + deadServer.getPublicAddress(), true); handleUserServerGroup(_serverGroupMap.get(deadServer.getGroup())); } } private static void handleUserServerGroup(ServerGroup serverGroup) { if (serverGroup != null && serverGroup.getHost() != null && !serverGroup.getHost().isEmpty() && serverGroup.getServerCount() == 0) { _repository.removeServerGroup(serverGroup); _serverGroupMap.remove(serverGroup); _serverGroups.remove(serverGroup); System.out.println("Removed ServerGroup : " + serverGroup.getName()); } } private static void calculateTotalPlayers() { for (ServerGroup serverGroup : _serverGroups) { _serverGroupMap.put(serverGroup.getName(), serverGroup); _totalPlayers += serverGroup.getPlayerCount(); } System.out.println("Total Players : " + _totalPlayers); } private static void handleGroupChanges(HashMap> serverTracker, ServerGroup serverGroup, boolean free) { int serverNum = 0; int requiredTotal = serverGroup.getRequiredTotalServers(); int requiredJoinable = serverGroup.getRequiredJoinableServers(); int joinableServers = serverGroup.getJoinableCount(); int totalServers = serverGroup.getServerCount(); int serversToAdd = Math.max(0, Math.max(requiredTotal - totalServers, requiredJoinable - joinableServers)); int serversToKill = (totalServers > requiredTotal && joinableServers > requiredJoinable) ? Math.min(joinableServers - requiredJoinable, serverGroup.getEmptyServers().size()) : 0; // Minimum 1500 slot bufferzone if (serverGroup.getName().equalsIgnoreCase("Lobby")) { int availableSlots = serverGroup.getMaxPlayerCount() - serverGroup.getPlayerCount(); if (availableSlots < 1500) { serversToAdd = Math.max(1, (1500 - availableSlots) / serverGroup.getMaxPlayers()); serversToKill = 0; } else if (serversToKill > 0) serversToKill = Math.min(serversToKill, (availableSlots - 1500) / 80); } else if (serverGroup.getName().equalsIgnoreCase("Halloween")) { if (serverGroup.getServers().size() > (_region == Region.US ? 300 : 100)) { serversToAdd = 0; } } else if (serverGroup.getName().equalsIgnoreCase("Christmas")) { if (serverGroup.getServers().size() > (_region == Region.US ? 300 : 100)) { serversToAdd = 0; } } else if (serverGroup.getName().equalsIgnoreCase("UltraHardcore")) { int maxUHC = Math.max(1, _totalPlayers / 6000); if (serversToAdd > 0) serversToAdd = maxUHC - joinableServers; if (joinableServers > maxUHC) serversToKill = maxUHC - joinableServers; } // KILL, CLEAN, THEN ADD while (serversToKill > 0) { List emptyServers = new ArrayList(serverGroup.getEmptyServers()); MinecraftServer emptyServer = emptyServers.get(serversToKill - 1); killServer(emptyServer, "[KILLED] [EXCESS] " + emptyServer.getName() + ":" + emptyServer.getPublicAddress()); serversToKill--; } while (serversToAdd > 0) { serverNum = serverGroup.generateUniqueId(serverNum + 1); Collections.sort(_dedicatedServers, new DedicatedServerSorter()); DedicatedServer bestServer = getBestDedicatedServer(_dedicatedServers, serverGroup); if (bestServer == null) { System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!! NO DEDICATED SERVER AVAILABLE FOR GROUP " + serverGroup.getName() + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!"); break; } if (serverTracker.containsKey(serverGroup.getPrefix() + "-" + serverNum)) System.out.println("[WAITING] On " + serverGroup.getPrefix() + "-" + serverNum + " to finish starting..."); else { startServer(bestServer, serverGroup, serverNum, free); serverTracker.put(serverGroup.getPrefix() + "-" + serverNum, new AbstractMap.SimpleEntry(bestServer.getPublicAddress(), System.currentTimeMillis())); } serversToAdd--; } } private static void killServer(final String serverName, final String serverAddress, final String message, final boolean announce) { if (_debug) return; String cmd = "/home/mineplex/easyRemoteKillServer.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverName}); pr.start(new GenericRunnable() { public void run(Boolean error) { if (!error) { MinecraftServer server = _repository.getServerStatus(serverName); if (server != null) { _repository.removeServerStatus(server); } } if (announce) { if (error) log("[" + serverName + ":" + serverAddress + "] Kill errored."); else log(message); } } }); try { pr.join(500); } catch (InterruptedException e1) { e1.printStackTrace(); } if (!pr.isDone()) _processes.add(pr); } private static boolean isServerOffline(DedicatedServer serverData) { boolean success = false; if (_debug) return false; Process process = null; String cmd = "/home/mineplex/isServerOnline.sh"; ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.getPublicAddress()}); try { process = processBuilder.start(); process.waitFor(); BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); String line = reader.readLine(); while(line != null) { success = line.equals("Success"); line=reader.readLine(); } } catch (Exception e1) { e1.printStackTrace(); } finally { process.destroy(); } return !success; } private static DedicatedServer getBestDedicatedServer(Collection dedicatedServers, ServerGroup serverGroup) { DedicatedServer bestServer = null; for (DedicatedServer serverData : dedicatedServers) { if (serverData.getAvailableRam() > serverGroup.getRequiredRam() && serverData.getAvailableCpu() > serverGroup.getRequiredCpu()) { if (bestServer == null || serverData.getServerCount(serverGroup) < bestServer.getServerCount(serverGroup)) { bestServer = serverData; } } } return bestServer; } private static void killServer(final MinecraftServer serverToKill, String message) { killServer(serverToKill.getName(), serverToKill.getPublicAddress(), message, true); } private static void startServer(final DedicatedServer serverSpace, final ServerGroup serverGroup, final int serverNum, final boolean free) { if (_debug) return; String cmd = "/home/mineplex/easyRemoteStartServerCustom.sh"; final String groupPrefix = serverGroup.getPrefix(); final String serverName = serverSpace.getName(); final String serverAddress = serverSpace.getPublicAddress(); ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverSpace.getPrivateAddress(), (serverGroup.getPortSection() + serverNum) + "", serverGroup.getRequiredRam() + "", serverGroup.getWorldZip(), serverGroup.getPlugin(), serverGroup.getConfigPath(), serverGroup.getName(), serverGroup.getPrefix() + "-" + serverNum, serverSpace.isUsRegion() ? "true" : "false", serverGroup.getAddNoCheat() + "" }); pr.start(new GenericRunnable() { public void run(Boolean error) { if (error) log("[" + serverName + ":" + serverAddress + " Free Resources; CPU " + serverSpace.getAvailableCpu() + " RAM " + serverSpace.getAvailableRam() + "MB] Errored " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")"); else log("[" + serverName + ":" + serverAddress + " Free Resources; CPU " + serverSpace.getAvailableCpu() + " RAM " + serverSpace.getAvailableRam() + "MB] Added " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")"); } }); try { pr.join(500); } catch (InterruptedException e1) { e1.printStackTrace(); } serverSpace.incrementServerCount(serverGroup); if (!pr.isDone()) _processes.add(pr); } private static void log(String message) { log(message, false); } private static void log(String message, boolean fileOnly) { _logger.info("[" + _dateFormat.format(new Date()) + "] " + message); if (!fileOnly) System.out.println("[" + _dateFormat.format(new Date()) + "] " + message); } }