Mineplex2018-withcommit/Plugins/Mineplex.ServerMonitor/src/mineplex/servermonitor/ServerMonitor.java

package mineplex.servermonitor;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.logging.FileHandler;
import java.util.logging.Logger;

import mineplex.core.common.util.NautHashMap;
import mineplex.serverdata.DedicatedServer;
import mineplex.serverdata.DedicatedServerSorter;
import mineplex.serverdata.MinecraftServer;
import mineplex.serverdata.Region;
import mineplex.serverdata.ServerGroup;
import mineplex.serverdata.ServerManager;
import mineplex.serverdata.ServerRepository;

public class ServerMonitor
{
	
	private static ServerRepository _repository = null;
	private static int _count = 0;
	private static HashSet<ProcessRunner> _processes = new HashSet<ProcessRunner>();
	private static HashMap<String, Boolean> _badServers = new HashMap<String, Boolean>();
	
	private static SimpleDateFormat _dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
	private static Logger _logger = Logger.getLogger("ServerMonitor");
	
	public static void main (String args[])
	{
		Region region = !new File("eu.dat").exists() ? Region.US : Region.EU;
		_repository = ServerManager.getServerRepository(region);	// Fetches and connects to server repo
		
		File logFile = new File("monitor.log");
		
		if (!logFile.exists())
		{
			try
			{
				logFile.createNewFile();
			}
			catch (IOException e1)
			{
				e1.printStackTrace();
			}
		}
		
		try
		{
			FileHandler fileHandler = new FileHandler("monitor.log");
			fileHandler.setFormatter(new CustomFormatter());
			_logger.addHandler(fileHandler);
			_logger.setUseParentHandlers(false);
		}
		catch (SecurityException | IOException e1)
		{
			e1.printStackTrace();
		}
		
		HashMap<String, Entry<String, Long>> serverTracker = new HashMap<String, Entry<String, Long>>();
		
		while (true)
		{
			Collection<ServerGroup> serverGroups = _repository.getServerGroups();
			
			for (MinecraftServer deadServer : _repository.getDeadServers())
			{
				killServer(deadServer.getName(), deadServer.getPublicAddress(), "[KILLED] [DEAD] " + deadServer.getName() + ":" + deadServer.getPublicAddress(), true);
			}
			
			List<DedicatedServer> dedicatedServers = new ArrayList<DedicatedServer>(_repository.getDedicatedServers());
			
			if (_count % 15 == 0)
			{				
				_badServers.clear();
				
				for (DedicatedServer serverData : dedicatedServers)
				{
					if (isServerOffline(serverData))
					{
						log("------=[OFFLINE]=------=[" + serverData.getName() + ":" + serverData.getPublicAddress() + "]=------=[OFFLINE]=------");
						_badServers.put(serverData.getName(), true);
					}
				}
				
				log(_badServers.size() + " bad servers.");
			}
			
			for (Iterator<DedicatedServer> iterator = dedicatedServers.iterator(); iterator.hasNext();)
			{
				DedicatedServer serverData = iterator.next();
				
				if (_badServers.containsKey(serverData.getName()))
					iterator.remove();
			}

			for (ServerGroup groupStatus : serverGroups)
			{
				NautHashMap<Integer, MinecraftServer> _serverMap = new NautHashMap<Integer, MinecraftServer>();
				
				for (Iterator<MinecraftServer> serverIterator = groupStatus.getServers().iterator(); serverIterator.hasNext();)
				{
					MinecraftServer server = serverIterator.next();
					int serverNum = Integer.parseInt(server.getName().split("-")[1]);
					
					if (_serverMap.containsKey(serverNum))
					{
						killServer(server.getName(), server.getPublicAddress(), "[KILLED] [DUPLICATE] " + server.getName() + ":" + server.getPublicAddress(), true);
						serverIterator.remove();
					}
					else
					{
						_serverMap.put(serverNum, server);
					}
				}
			}
			
			HashSet<String> onlineServers = new HashSet<String>();
			
			for (MinecraftServer minecraftServer : _repository.getServerStatuses())
			{
				onlineServers.add(minecraftServer.getName());
				
				if (minecraftServer.getTps() <= 17)
					log("[Performance] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress() + "] Running poorly at " + minecraftServer.getTps() + " TPS");
			}
			
			for (Iterator<Entry<String, Entry<String, Long>>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();)
			{
				Entry<String, Entry<String, Long>> entry = iterator.next();
				
				if (onlineServers.contains(entry.getKey()))
					iterator.remove();
				else if (System.currentTimeMillis() - entry.getValue().getValue() > 20000)
				{	
					String serverName = entry.getKey();
					String serverAddress = entry.getValue().getKey();
					killServer(serverName, serverAddress, "[KILLED] [SLOW-STARTUP] " + serverName + ":" + serverAddress, true);
					iterator.remove();
				}
			}
			
			for (ServerGroup serverGroup : serverGroups)
			{
				handleGroupChanges(dedicatedServers, serverTracker, serverGroup, false);
			}
		
			int processWaits = 0;
			
			while (_processes.size() > 0)
			{
				for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
				{
					ProcessRunner pr = iterator.next();
					
					try 
					{
						pr.join(100);
					} 
					catch (InterruptedException e) 
					{
						e.printStackTrace();
					}
					
					if (pr.isDone())
						iterator.remove();
				}
				
				if (_processes.size() > 0)
				{
					try
					{
						log("Sleeping while processes run...");
						Thread.sleep(6000);
					} 
					catch (InterruptedException e)
					{
						e.printStackTrace();
					}
				}
				
				if (processWaits >= 10)
				{
					log("Killing stale processes.");
					
					for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
					{
						iterator.next().abort();
						iterator.remove();
					}
				}
				
				processWaits++;
			}
		
			processWaits = 0;
			
			try
			{
				log("Natural sleep.");
				Thread.sleep(10000);
			} 
			catch (InterruptedException e)
			{
				e.printStackTrace();
			}
			
			_count++;
		}
	}

	private static void handleGroupChanges(List<DedicatedServer> dedicatedServers, HashMap<String, Entry<String, Long>> serverTracker, ServerGroup serverGroup, boolean free)
	{
		int serverNum = 0;
		int requiredTotal = serverGroup.getRequiredTotalServers();
		int requiredJoinable = serverGroup.getRequiredJoinableServers();
		int joinableServers = serverGroup.getJoinableCount();
		int totalServers = serverGroup.getServerCount();
		int serversToAdd = Math.max(requiredTotal - totalServers, requiredJoinable - joinableServers);
		int serversToKill = (totalServers > requiredTotal && joinableServers > requiredJoinable) ? Math.min(joinableServers - requiredJoinable, serverGroup.getEmptyServers().size()) : 0;

		// Minimum 1500 slot bufferzone
		if (serverGroup.getName().equalsIgnoreCase("Lobby"))
		{
			int availableSlots = serverGroup.getMaxPlayerCount() - serverGroup.getPlayerCount();
			
			if (availableSlots < 1500)
				serversToAdd = Math.max(1, (1500 - availableSlots) / serverGroup.getMaxPlayers());
			else if (serversToKill > 0)
				serversToKill = Math.min(serversToKill, (availableSlots - 1500) / 80);
		}
		
		while (serversToAdd > 0)
		{
			serverNum = serverGroup.generateUniqueId(serverNum + 1);
			Collections.sort(dedicatedServers, new DedicatedServerSorter());
			DedicatedServer bestServer = getBestDedicatedServer(dedicatedServers, serverGroup);
			
			if (bestServer == null)
			{
				System.out.println("No best dynamic server available for group " + serverGroup.getName());
				break;
			}

			if (serverTracker.containsKey(serverGroup.getPrefix() + "-" + serverNum))
				System.out.println("[WAITING] On " + serverGroup.getPrefix() + "-" + serverNum + " to finish starting...");
			else
			{
				startServer(bestServer, serverGroup, serverNum, free);
				serverTracker.put(serverGroup.getPrefix() + "-" + serverNum, new AbstractMap.SimpleEntry<String, Long>(bestServer.getPublicAddress(), System.currentTimeMillis()));
			}
			
		    serversToAdd--;
		}
		
		while (serversToKill > 0)
		{
			List<MinecraftServer> emptyServers = new ArrayList<MinecraftServer>(serverGroup.getEmptyServers());
			MinecraftServer emptyServer = emptyServers.get(serversToKill - 1);
			killServer(emptyServer, "[KILLED] [EXCESS] " + emptyServer.getName() + ":" + emptyServer.getPublicAddress());
		    serversToKill--;
		}
	}

	private static void killServer(final String serverName, final String serverAddress, final String message, final boolean announce)
	{
		String cmd = "/home/mineplex/easyRemoteKillServer.sh";
		
		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverName});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (!error)
				{
					MinecraftServer server = _repository.getServerStatus(serverName);
					
					if (server != null)
					{
						_repository.removeServerStatus(server);
					}
				}
				
				if (announce)
				{
					if (error)
						log("[" + serverName + ":" + serverAddress + "] Kill errored.");
					else
						log(message);
				}
			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		
		if (!pr.isDone())
			_processes.add(pr);
	}

	private static boolean isServerOffline(DedicatedServer serverData)
	{
		boolean success = false;
		
		Process process = null;
		String cmd = "/home/mineplex/isServerOnline.sh";
		
		ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.getPublicAddress()});
	
		try
		{
			process = processBuilder.start();
			process.waitFor();
			
            BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream())); 
            String line = reader.readLine(); 
            
            while(line != null) 
            {
            	success = line.equals("Success");
            	
	            line=reader.readLine(); 
            } 
        } 
		catch (Exception e1) 
		{
			e1.printStackTrace();
		}
		finally
		{
			process.destroy();
		}

		return !success;
	}

	private static DedicatedServer getBestDedicatedServer(Collection<DedicatedServer> dedicatedServers,	ServerGroup serverGroup)
	{
		DedicatedServer bestServer = null;
		
		for (DedicatedServer serverData : dedicatedServers)
		{
			if (serverData.getAvailableRam() > serverGroup.getRequiredRam() 
					&& serverData.getAvailableCpu() > serverGroup.getRequiredCpu())
			{
				if (bestServer == null 
					|| serverData.getServerCount(serverGroup) < bestServer.getServerCount(serverGroup))
				{
					bestServer = serverData;
				}
			}
		}
		
		return bestServer;
	}

	private static void killServer(final MinecraftServer serverToKill, String message)
	{
		killServer(serverToKill.getName(), serverToKill.getPublicAddress(), message, true);
	}
	
	private static void startServer(final DedicatedServer serverSpace, final ServerGroup serverGroup, final int serverNum, final boolean free)
	{
		String cmd = "/home/mineplex/easyRemoteStartServer.sh";
		final String groupPrefix = serverGroup.getPrefix();
		final String serverName = serverSpace.getName();
		final String serverAddress = serverSpace.getPublicAddress();

		ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverSpace.getPrivateAddress(), (serverGroup.getPortSection() + serverNum) + "", serverGroup.getRequiredRam() + "", serverGroup.getWorldZip(), serverGroup.getPlugin(), serverGroup.getConfigPath(), serverGroup.getName(), serverGroup.getPrefix() + "-" + serverNum, serverGroup.getMinPlayers() + "", serverGroup.getMaxPlayers() + "", serverGroup.getPvp() + "", serverGroup.getTournament() + "", free + "", serverSpace.isUsRegion() ? "true" : "false", serverGroup.getArcadeGroup() + "", serverGroup.getGames(), serverGroup.getServerType(), serverGroup.getAddNoCheat() + "", serverGroup.getTeamAutoJoin() + "", serverGroup.getTeamForceBalance() + "", serverGroup.getTeamRejoin() + "", serverGroup.getGameAutoStart() + "", serverGroup.getGameTimeout() + "", serverGroup.getHotbarHubClock() + "", serverGroup.getHotbarInventory() + "", serverGroup.getPlayerKickIdle() + "", serverGroup.getRewardGems() + "", serverGroup.getRewardItems() + "", serverGroup.getRewardAchievements() + "", serverGroup.getRewardStats() + ""});
		pr.start(new GenericRunnable<Boolean>()		
		{
			public void run(Boolean error)
			{
				if (error)
					log("[" + serverName + ":" + serverAddress + "] Errored " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")");
				else
					log("[" + serverName + ":" + serverAddress + "] Added " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")");

			}
		});
		
		try 
		{
			pr.join(500);
		} 
		catch (InterruptedException e1) 
		{
			e1.printStackTrace();
		}
		
		serverSpace.incrementServerCount(serverGroup);
		
		if (!pr.isDone())
			_processes.add(pr);
	}
	
	private static void log(String message)
	{
		_logger.info("[" + _dateFormat.format(new Date()) + "] " + message);
		System.out.println("[" + _dateFormat.format(new Date()) + "] " + message);
	}
}