package mineplex.servermonitor; import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; public class ServerMonitor { private static boolean _us = true; private static Repository _repository = new Repository(); private static int _count = 0; private static HashSet _processes = new HashSet(); private static HashMap _badServers = new HashMap(); public static void main (String args[]) { _us = !new File("eu.dat").exists(); _repository.initialize(_us); HashMap> serverTracker = new HashMap>(); while (true) { Collection serverGroups = _repository.retrieveServerGroups(); HashMap groupStatusList = _repository.retrieveGroupStatusData(); for (ServerStatusData statusData : _repository.retrieveOldServerStatuses()) { /* if (us) { if (!serverTracker.containsKey(statusData.Name)) { restartServer(statusData); serverTracker.put(statusData.Name, new AbstractMap.SimpleEntry(statusData.Address, System.currentTimeMillis())); } else if (System.currentTimeMillis() - serverTracker.get(statusData.Name).getValue() > 15000) { serverTracker.remove(statusData.Name); System.out.println("-=[SERVER RESTART TOO SLOW]=- " + statusData.Name + " (Putting back in restart queue)"); } } else */ killServer(statusData); } /* if (us) { // Remove successfully restarted US servers for (GroupStatusData groupStatus : groupStatusList.values()) { for (ServerStatusData serverToKill : groupStatus.Servers.values()) { if (serverTracker.containsKey(serverToKill.Name)) serverTracker.remove(serverToKill.Name); } } } else { */ List dynamicServers = new ArrayList(_repository.retrieveDynamicServers()); if (_count % 15 == 0) { _badServers.clear(); for (DynamicServerData serverData : dynamicServers) { if (isServerOffline(serverData)) { System.out.println("------=[OFFLINE]=------=[" + serverData.Name + ":" + serverData.Address + "]=------=[OFFLINE]=------"); _badServers.put(serverData.Name, true); } } System.out.println(_badServers.size() + " bad servers."); } for (Iterator iterator = dynamicServers.iterator(); iterator.hasNext();) { DynamicServerData serverData = iterator.next(); if (_badServers.containsKey(serverData.Name)) iterator.remove(); } for (GroupStatusData groupStatus : groupStatusList.values()) { for (ServerStatusData serverToKill : groupStatus.KillServers) { System.out.println("----DUPLICATE SERVER----> " + serverToKill.Address + ", " + serverToKill.Name); killServer(serverToKill); } for (ServerStatusData serverToKill : groupStatus.Servers.values()) { if (serverTracker.containsKey(serverToKill.Name)) serverTracker.remove(serverToKill.Name); } } for (Iterator>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();) { Entry> entry = iterator.next(); if (System.currentTimeMillis() - entry.getValue().getValue() > 15000) { System.out.println("-=[SERVER STARTUP TOO SLOW]=- " + entry.getKey()); ServerStatusData serverToKill = new ServerStatusData(); serverToKill.Name = entry.getKey(); serverToKill.Address = entry.getValue().getKey(); killServer(serverToKill); iterator.remove(); } } for (ServerGroupData serverGroup : serverGroups) { if (!groupStatusList.containsKey(serverGroup.Name)) { groupStatusList.put(serverGroup.Name, new GroupStatusData()); } GroupStatusData groupStatus = groupStatusList.get(serverGroup.Name); int serversToAdd = Math.max(serverGroup.RequiredTotalServers - groupStatus.getTotalServers(), serverGroup.RequiredJoinableServers - groupStatus.getJoinableCount()); int serversToKill = (groupStatus.getTotalServers() > serverGroup.RequiredTotalServers && groupStatus.getJoinableCount() > serverGroup.RequiredJoinableServers) ? Math.min(groupStatus.getJoinableCount() - serverGroup.RequiredJoinableServers, groupStatus.EmptyServers.size()) : 0; while (serversToAdd > 0) { int serverNum = groupStatus.getNextServerNumber(); Collections.sort(dynamicServers, new DynamicServerSorter()); DynamicServerData bestServer = getBestDynamicServer(dynamicServers, serverGroup); if (bestServer == null) { System.out.println("No best dynamic server available for group " + serverGroup.Name); break; } if (serverTracker.containsKey(serverGroup.Prefix + "-" + serverNum)) System.out.println("[WAITING] On " + serverGroup.Prefix + "-" + serverNum + " to finish starting..."); else { startServer(bestServer, serverGroup, serverNum); serverTracker.put(serverGroup.Prefix + "-" + serverNum, new AbstractMap.SimpleEntry(bestServer.Address, System.currentTimeMillis())); } serversToAdd--; } while (serversToKill > 0) { System.out.println("[" + groupStatus.EmptyServers.get(0).Name + ":" + groupStatus.EmptyServers.get(0).Address + "] Killing " + serverGroup.Name + " Req Total: " + serverGroup.RequiredTotalServers + " Req Joinable: " + serverGroup.RequiredJoinableServers + " | Actual Total: " + groupStatus.getTotalServers() + " Actual Joinable: " + groupStatus.getJoinableCount()); killServer(groupStatus.EmptyServers.remove(0)); serversToKill--; } } //} int processWaits = 0; while (_processes.size() > 0) { for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { ProcessRunner pr = iterator.next(); try { pr.join(100); } catch (InterruptedException e) { e.printStackTrace(); } if (pr.isDone()) iterator.remove(); } if (_processes.size() > 0) { try { System.out.println("Sleeping while processes run..."); Thread.sleep(6000); } catch (InterruptedException e) { e.printStackTrace(); } } if (processWaits >= 10) { System.out.println("Killing stale processes."); for (Iterator iterator = _processes.iterator(); iterator.hasNext();) { iterator.next().abort(); iterator.remove(); } } processWaits++; } processWaits = 0; try { System.out.println("Natural sleep."); Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); } _count++; } } private static void killServer(final ServerStatusData serverToKill, final boolean announce) { String cmd = "/home/mineplex/easyRemoteKillServer.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name}); pr.start(new GenericRunnable() { public void run(Boolean error) { if (!error) _repository.removeServerRecord(serverToKill); if (announce) { if (error) System.out.println("[" + serverToKill.Name + ":" + serverToKill.Address + "] Kill errored."); else System.out.println("Sent kill command to " + serverToKill.Address + " for " + serverToKill.Name + " completed"); } } }); try { pr.join(500); } catch (InterruptedException e1) { e1.printStackTrace(); } if (!pr.isDone()) _processes.add(pr); } private static boolean isServerOffline(DynamicServerData serverData) { boolean success = false; Process process = null; String cmd = "/home/mineplex/isServerOnline.sh"; ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.Address}); try { process = processBuilder.start(); process.waitFor(); BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); String line = reader.readLine(); while(line != null) { success = line.equals("Success"); line=reader.readLine(); } } catch (Exception e1) { e1.printStackTrace(); } finally { process.destroy(); } return !success; } private static DynamicServerData getBestDynamicServer(Collection dynamicServers, ServerGroupData serverGroup) { DynamicServerData bestServer = null; for (DynamicServerData serverData : dynamicServers) { if (serverData.AvailableRAM > serverGroup.RequiredRAM && serverData.AvailableCPU > serverGroup.RequiredCPU) { if (bestServer == null) { bestServer = serverData; if (!serverData.ServerGroupCount.containsKey(serverGroup.Name)) break; } else if (serverData.ServerGroupCount.containsKey(serverGroup.Name)) { if (serverData.ServerGroupCount.get(serverGroup.Name) < bestServer.ServerGroupCount.get(serverGroup.Name)) bestServer = serverData; } } } return bestServer; } private static void restartServer(final ServerStatusData serverToKill) { String cmd = "/home/mineplex/restartServer.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name}); pr.start(new GenericRunnable() { public void run(Boolean error) { if (error) System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " failed"); else System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " completed"); } }); try { pr.join(500); } catch (InterruptedException e1) { e1.printStackTrace(); } if (!pr.isDone()) _processes.add(pr); } private static void killServer(final ServerStatusData serverToKill) { killServer(serverToKill, true); } private static void startServer(final DynamicServerData serverSpace, final ServerGroupData serverGroup, final int serverNum) { String cmd = "/home/mineplex/easyRemoteStartServer.sh"; ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverSpace.Address, serverSpace.PrivateAddress, serverGroup.ScriptName, serverGroup.Prefix + "-" + serverNum, "1", serverSpace.US ? "us" : "eu"}); pr.start(new GenericRunnable() { public void run(Boolean error) { if (error) System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Errored " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")"); else System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Added " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")"); } }); try { pr.join(500); } catch (InterruptedException e1) { e1.printStackTrace(); } serverSpace.setServerGroupCount(serverGroup, serverSpace.ServerGroupCount.containsKey(serverGroup.Name) ? (serverSpace.ServerGroupCount.get(serverGroup.Name) + 1) : 1); if (!pr.isDone()) _processes.add(pr); } }