Mineplex2018-withcommit/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java

package mineplex.servermonitor;

import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;

public class ServerMonitor
{
	private static boolean _us = true;
	private static Repository _repository = new Repository();
	private static int _count = 0;
	private static HashSet<ProcessRunner> _processes = new HashSet<ProcessRunner>();
	private static HashMap<String, Boolean> _badServers = new HashMap<String, Boolean>();
	
	public static void main (String args[])
	{
		_us = !new File("eu.dat").exists();
		
		_repository.initialize(_us);
		HashMap<String, Entry<String, Long>> serverTracker = new HashMap<String, Entry<String, Long>>();
		
		while (true)
		{
			Collection<ServerGroupData> serverGroups = _repository.retrieveServerGroups();
			HashMap<String, GroupStatusData> groupStatusList = _repository.retrieveGroupStatusData();
			
			for (ServerStatusData statusData : _repository.retrieveOldServerStatuses())
			{
				/*
				if (us)
				{
					if (!serverTracker.containsKey(statusData.Name))
					{
						restartServer(statusData);
						serverTracker.put(statusData.Name, new AbstractMap.SimpleEntry<String, Long>(statusData.Address, System.currentTimeMillis()));
					}
					else if (System.currentTimeMillis() - serverTracker.get(statusData.Name).getValue() > 15000)
					{
						serverTracker.remove(statusData.Name);
						System.out.println("-=[SERVER RESTART TOO SLOW]=- " + statusData.Name + " (Putting back in restart queue)");
					}
				}
				else
				*/
				
				killServer(statusData);
			}
			
			/*
			if (us)
			{
				// Remove successfully restarted US servers
				for (GroupStatusData groupStatus : groupStatusList.values())
				{
					for (ServerStatusData serverToKill : groupStatus.Servers.values())
					{
						if (serverTracker.containsKey(serverToKill.Name))
							serverTracker.remove(serverToKill.Name);
					}
				}
			}
			else
			{
			*/
				List<DynamicServerData> dynamicServers = new ArrayList<DynamicServerData>(_repository.retrieveDynamicServers());
				
				if (_count % 15 == 0)
				{				
					_badServers.clear();
					
					for (DynamicServerData serverData : dynamicServers)
					{					
						if (isServerOffline(serverData))
						{
							System.out.println("------=[OFFLINE]=------=[" + serverData.Name + ":" + serverData.Address + "]=------=[OFFLINE]=------");
							_badServers.put(serverData.Name, true);
						}
					}
					
					System.out.println(_badServers.size() + " bad servers.");
				}
				
				for (Iterator<DynamicServerData> iterator = dynamicServers.iterator(); iterator.hasNext();)
				{
					DynamicServerData serverData = iterator.next();
					
					if (_badServers.containsKey(serverData.Name))
						iterator.remove();
				}
	
				for (GroupStatusData groupStatus : groupStatusList.values())
				{
					for (ServerStatusData serverToKill : groupStatus.KillServers)
					{
						System.out.println("----DUPLICATE SERVER----> " + serverToKill.Address + ", " + serverToKill.Name);
						killServer(serverToKill);
					}
					
					for (ServerStatusData serverToKill : groupStatus.Servers.values())
					{
						if (serverTracker.containsKey(serverToKill.Name))
							serverTracker.remove(serverToKill.Name);
					}
				}
				
				for (Iterator<Entry<String, Entry<String, Long>>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();)
				{
					Entry<String, Entry<String, Long>> entry = iterator.next();
					
					if (System.currentTimeMillis() - entry.getValue().getValue() > 15000)
					{	
						System.out.println("-=[SERVER STARTUP TOO SLOW]=- " + entry.getKey());
						
						ServerStatusData serverToKill = new ServerStatusData();
						serverToKill.Name = entry.getKey();
						serverToKill.Address = entry.getValue().getKey();
						
						killServer(serverToKill);
						iterator.remove();
					}
				}
				
				for (ServerGroupData serverGroup : serverGroups)
				{
					if (!groupStatusList.containsKey(serverGroup.Name))
					{
						groupStatusList.put(serverGroup.Name, new GroupStatusData());
					}
				
					GroupStatusData groupStatus = groupStatusList.get(serverGroup.Name);
					
					int serversToAdd = Math.max(serverGroup.RequiredTotalServers - groupStatus.getTotalServers(), serverGroup.RequiredJoinableServers - groupStatus.getJoinableCount());
					int serversToKill = (groupStatus.getTotalServers() > serverGroup.RequiredTotalServers && groupStatus.getJoinableCount() > serverGroup.RequiredJoinableServers) ? Math.min(groupStatus.getJoinableCount() - serverGroup.RequiredJoinableServers, groupStatus.EmptyServers.size()) : 0;
	
					while (serversToAdd > 0)
					{
						int serverNum = groupStatus.getNextServerNumber();
						Collections.sort(dynamicServers, new DynamicServerSorter());
						DynamicServerData bestServer = getBestDynamicServer(dynamicServers, serverGroup);
						
						if (bestServer == null)
						{
							System.out.println("No best dynamic server available for group " + serverGroup.Name);
							break;
						}
	
						if (serverTracker.containsKey(serverGroup.Prefix + "-" + serverNum))
							System.out.println("[WAITING] On " + serverGroup.Prefix + "-" + serverNum + " to finish starting...");
						else
						{
							startServer(bestServer, serverGroup, serverNum);
							serverTracker.put(serverGroup.Prefix + "-" + serverNum, new AbstractMap.SimpleEntry<String, Long>(bestServer.Address, System.currentTimeMillis()));
						}
						
					    serversToAdd--;
					}
					
					while (serversToKill > 0)
					{
						System.out.println("[" + groupStatus.EmptyServers.get(0).Name + ":" + groupStatus.EmptyServers.get(0).Address + "] Killing " + serverGroup.Name + " Req Total: " + serverGroup.RequiredTotalServers + " Req Joinable: " + serverGroup.RequiredJoinableServers + " | Actual Total: " + groupStatus.getTotalServers() + " Actual Joinable: " + groupStatus.getJoinableCount());
						killServer(groupStatus.EmptyServers.remove(0));
					    serversToKill--;
					}
				}
			//}
		
			int processWaits = 0;
			
			while (_processes.size() > 0)
			{
				for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
				{
					ProcessRunner pr = iterator.next();
					
					try 
					{
						pr.join(100);
					} 
					catch (InterruptedException e) 
					{
						e.printStackTrace();
					}
					
					if (pr.isDone())
						iterator.remove();
				}
				
				if (_processes.size() > 0)
				{
					try
					{
						System.out.println("Sleeping while processes run...");
						Thread.sleep(6000);
					} 
					catch (InterruptedException e)
					{
						e.printStackTrace();
					}
				}
				
				if (processWaits >= 10)
				{
					System.out.println("Killing stale processes.");
					
					for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
					{
						iterator.next().abort();
						iterator.remove();
					}
				}
				
				processWaits++;
			}
		
			processWaits = 0;	
			
			try
			{
				System.out.println("Natural sleep.");
				Thread.sleep(10000);
			} 
			catch (InterruptedException e)
			{
				e.printStackTrace();
			}
			
			_count++;
		}
	}

	private static void killServer(final ServerStatusData serverToKill, final boolean announce)
	{
		String cmd = "/home/mineplex/easyRemoteKillServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (!error)
					_repository.removeServerRecord(serverToKill);
				
				if (announce)
				{
					if (error)
						System.out.println("[" + serverToKill.Name + ":" + serverToKill.Address + "] Kill errored.");
					else
						System.out.println("Sent kill command to " + serverToKill.Address + " for " + serverToKill.Name + " completed");
				}
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		
		if (!pr.isDone())
			_processes.add(pr);
	}

	private static boolean isServerOffline(DynamicServerData serverData)
	{
		boolean success = false;
		
		Process process = null;
		String cmd = "/home/mineplex/isServerOnline.sh";
		
		ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.Address});
	
		try
		{
			process = processBuilder.start();
			process.waitFor();
			
            BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); 
            String line = reader.readLine(); 
            
            while(line != null) 
            {
            	success = line.equals("Success");
            	
	            line=reader.readLine(); 
            } 
        } 
		catch (Exception e1) 
		{
			e1.printStackTrace();
		}
		finally
		{
			process.destroy();
		}

		return !success;
	}

	private static DynamicServerData getBestDynamicServer(Collection<DynamicServerData> dynamicServers,	ServerGroupData serverGroup)
	{
		DynamicServerData bestServer = null;
		
		for (DynamicServerData serverData : dynamicServers)
		{
			if (serverData.AvailableRAM > serverGroup.RequiredRAM && serverData.AvailableCPU > serverGroup.RequiredCPU)
			{
				if (bestServer == null)
				{
					bestServer = serverData;
					
					if (!serverData.ServerGroupCount.containsKey(serverGroup.Name))
						break;
				}
				else if (serverData.ServerGroupCount.containsKey(serverGroup.Name))					
				{
					if (serverData.ServerGroupCount.get(serverGroup.Name) < bestServer.ServerGroupCount.get(serverGroup.Name))
						bestServer = serverData;
				}
			}
		}
		
		return bestServer;
	}
	
	private static void restartServer(final ServerStatusData serverToKill)
	{
		String cmd = "/home/mineplex/restartServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (error)
					System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " failed");
				else
					System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " completed");
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		if (!pr.isDone())
			_processes.add(pr);
	}
	
	private static void killServer(final ServerStatusData serverToKill)
	{
		killServer(serverToKill, true);
	}
	
	private static void startServer(final DynamicServerData serverSpace, final ServerGroupData serverGroup, final int serverNum)
	{
		String cmd = "/home/mineplex/easyRemoteStartServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverSpace.Address, serverSpace.PrivateAddress, serverGroup.ScriptName, serverGroup.Prefix + "-" + serverNum, "1", serverSpace.US ? "us" : "eu"});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (error)
					System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Errored " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")");
				else
					System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Added " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")");
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		serverSpace.setServerGroupCount(serverGroup, serverSpace.ServerGroupCount.containsKey(serverGroup.Name) ? (serverSpace.ServerGroupCount.get(serverGroup.Name) + 1) : 1);
		
		if (!pr.isDone())
			_processes.add(pr);
	}
}