Mineplex2018-withcommit/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java

package mineplex.servermonitor;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

public class ServerMonitor
{
	private static Repository _repository = new Repository();
	private static int _count = 0;
	private static HashSet<ProcessRunner> _processes = new HashSet<ProcessRunner>();
	private static HashMap<String, Boolean> _badServers = new HashMap<String, Boolean>();
	
	public static void main (String args[])
	{		
		_repository.initialize();
		
		while (true)
		{			
			for (ServerStatusData statusData : _repository.retrieveOldServerStatuses())
			{
				System.out.println("----Old Server Status----> " + statusData.Address + ", " + statusData.Name);
				killServer(statusData);
				//restartServer(statusData);
			}
			
			List<DynamicServerData> dynamicServers = new ArrayList<DynamicServerData>(_repository.retrieveDynamicServers());
			
			if (_count % 15 == 0)
			{				
				_badServers.clear();
				
				for (DynamicServerData serverData : dynamicServers)
				{					
					if (isServerOffline(serverData))
					{
						System.out.println("------=[OFFLINE]=------=[" + serverData.Name + ":" + serverData.Address + "]=------=[OFFLINE]=------");
						_badServers.put(serverData.Name, true);
					}
				}
				
				System.out.println(_badServers.size() + " bad servers.");
			}
			
			for (Iterator<DynamicServerData> iterator = dynamicServers.iterator(); iterator.hasNext();)
			{
				DynamicServerData serverData = iterator.next();
				
				if (_badServers.containsKey(serverData.Name))
					iterator.remove();
			}
			
			Collection<ServerGroupData> serverGroups = _repository.retrieveServerGroups();
			HashMap<String, GroupStatusData> groupStatusList = _repository.retrieveGroupStatusData();
			
			for (ServerGroupData serverGroup : serverGroups)
			{
				if (!groupStatusList.containsKey(serverGroup.Name))
				{
					groupStatusList.put(serverGroup.Name, new GroupStatusData());
				}
			
				GroupStatusData groupStatus = groupStatusList.get(serverGroup.Name);
				
				int serversToAdd = Math.max(serverGroup.RequiredTotalServers - groupStatus.getTotalServers(), serverGroup.RequiredJoinableServers - groupStatus.getJoinableCount());
				int serversToKill = (groupStatus.getTotalServers() > serverGroup.RequiredTotalServers && groupStatus.getJoinableCount() > serverGroup.RequiredJoinableServers) ? Math.min(groupStatus.getJoinableCount() - serverGroup.RequiredJoinableServers, groupStatus.EmptyServers.size()) : 0;

				while (serversToAdd > 0)
				{
					int serverNum = groupStatus.getNextServerNumber();
					Collections.sort(dynamicServers, new DynamicServerSorter());
					DynamicServerData bestServer = getBestDynamicServer(dynamicServers, serverGroup);
					
					if (bestServer == null)
					{
						System.out.println("No best dynamic server available for group " + serverGroup.Name);
						break;
					}

					System.out.println("[" + bestServer.Name + ":" + bestServer.Address + "] Adding " + serverGroup.Name + " Req Total: " + serverGroup.RequiredTotalServers + " Req Joinable: " + serverGroup.RequiredJoinableServers + " | Actual Total: " + groupStatus.getTotalServers() + " Actual Joinable: " + groupStatus.getJoinableCount());
					
					/*
					// Kill if any previous attempts stack up
					ServerStatusData newServer = new ServerStatusData();
					newServer.Address = bestServer.Address;
					newServer.Name = serverGroup.Prefix + "-" + serverNum;
					killServer(newServer, false);
					
					// Delay between kill and start so we don't kill the new process
					try
					{
						Thread.sleep(100);
					}
					catch (InterruptedException e)
					{
						e.printStackTrace();
					}
					*/
					
					startServer(bestServer, serverGroup, serverNum);
					
				    serversToAdd--;
				}
				
				while (serversToKill > 0)
				{
					System.out.println("[" + groupStatus.EmptyServers.get(0).Name + ":" + groupStatus.EmptyServers.get(0).Address + "] Killing " + serverGroup.Name + " Req Total: " + serverGroup.RequiredTotalServers + " Req Joinable: " + serverGroup.RequiredJoinableServers + " | Actual Total: " + groupStatus.getTotalServers() + " Actual Joinable: " + groupStatus.getJoinableCount());
					killServer(groupStatus.EmptyServers.remove(0));
				    serversToKill--;
				}
			}
			
			int processWaits = 0;
			
			while (_processes.size() > 0)
			{
				for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
				{
					ProcessRunner pr = iterator.next();
					
					try 
					{
						pr.join(100);
					} 
					catch (InterruptedException e) 
					{
						e.printStackTrace();
					}
					
					if (pr.isDone())
						iterator.remove();
				}
				
				if (_processes.size() > 0)
				{
					try
					{
						System.out.println("Sleeping while processes run...");
						Thread.sleep(6000);
					} 
					catch (InterruptedException e)
					{
						e.printStackTrace();
					}
				}
				
				if (processWaits >= 10)
				{
					System.out.println("Killing stale processes.");
					
					for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
					{
						iterator.next().abort();
						iterator.remove();
					}
				}
				
				processWaits++;
			}
			
			processWaits = 0;	
			
			try
			{
				System.out.println("Natural sleep.");
				Thread.sleep(5000);
			} 
			catch (InterruptedException e)
			{
				e.printStackTrace();
			}
			
			_count++;
		}
	}

	private static void killServer(final ServerStatusData serverToKill, final boolean announce)
	{
		String cmd = "/home/mineplex/easyRemoteKillServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (!error)
					_repository.removeServerRecord(serverToKill);
				
				if (announce)
				{
					if (error)
						System.out.println("[" + serverToKill.Name + ":" + serverToKill.Address + "] Kill errored.");
					else
						System.out.println("Sent kill command to " + serverToKill.Address + " for " + serverToKill.Name + " completed");
				}
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		
		if (!pr.isDone())
			_processes.add(pr);
	}

	private static boolean isServerOffline(DynamicServerData serverData)
	{
		boolean success = false;
		
		Process process = null;
		String cmd = "/home/mineplex/isServerOnline.sh";
		
		ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.Address});
	
		try
		{
			process = processBuilder.start();
			process.waitFor();
			
            BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); 
            String line = reader.readLine(); 
            
            while(line != null) 
            {
            	success = line.equals("Success");
            	
	            line=reader.readLine(); 
            } 
        } 
		catch (Exception e1) 
		{
			e1.printStackTrace();
		}
		finally
		{
			process.destroy();
		}

		return !success;
	}

	private static DynamicServerData getBestDynamicServer(Collection<DynamicServerData> dynamicServers,	ServerGroupData serverGroup)
	{
		DynamicServerData bestServer = null;
		
		for (DynamicServerData serverData : dynamicServers)
		{
			if (serverData.AvailableRAM > serverGroup.RequiredRAM && serverData.AvailableCPU > serverGroup.RequiredCPU)
			{
				if (bestServer == null)
				{
					bestServer = serverData;
					
					if (!serverData.ServerGroupCount.containsKey(serverGroup.Name))
						break;
				}
				else if (serverData.ServerGroupCount.containsKey(serverGroup.Name))					
				{
					if (serverData.ServerGroupCount.get(serverGroup.Name) < bestServer.ServerGroupCount.get(serverGroup.Name))
						bestServer = serverData;
				}
			}
		}
		
		return bestServer;
	}
	
	private static void restartServer(final ServerStatusData serverToKill)
	{
		String cmd = "/home/mineplex/restartServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverToKill.Address, serverToKill.Name});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (error)
					System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " failed");
				else
					System.out.println("Restart command to " + serverToKill.Address + " for " + serverToKill.Name + " completed");
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		if (!pr.isDone())
			_processes.add(pr);
	}
	
	private static void killServer(final ServerStatusData serverToKill)
	{
		killServer(serverToKill, true);
	}
	
	private static void startServer(final DynamicServerData serverSpace, final ServerGroupData serverGroup, final int serverNum)
	{
		String cmd = "/home/mineplex/easyRemoteStartServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverSpace.Address, serverSpace.PrivateAddress, serverGroup.ScriptName, serverGroup.Prefix + "-" + serverNum, "1", serverSpace.US ? "us" : "eu"});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (error)
					System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Errored " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")");
				else
					System.out.println("[" + serverSpace.Name + ":" + serverSpace.Address + "] Added " + serverGroup.Name + "(" + serverGroup.Prefix + "-" + serverNum + ")");
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		serverSpace.setServerGroupCount(serverGroup, serverSpace.ServerGroupCount.containsKey(serverGroup.Name) ? (serverSpace.ServerGroupCount.get(serverGroup.Name) + 1) : 1);
		
		if (!pr.isDone())
			_processes.add(pr);
	}
}