473 lines
14 KiB
Java
473 lines
14 KiB
Java
package mineplex.servermonitor;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.AbstractMap;
|
|
import java.util.ArrayList;
|
|
import java.util.Collection;
|
|
import java.util.Collections;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
import java.util.logging.FileHandler;
|
|
import java.util.logging.Logger;
|
|
|
|
import mineplex.core.common.util.NautHashMap;
|
|
import mineplex.serverdata.DedicatedServer;
|
|
import mineplex.serverdata.DedicatedServerSorter;
|
|
import mineplex.serverdata.MinecraftServer;
|
|
import mineplex.serverdata.Region;
|
|
import mineplex.serverdata.ServerGroup;
|
|
import mineplex.serverdata.ServerManager;
|
|
import mineplex.serverdata.ServerRepository;
|
|
|
|
public class ServerMonitor
|
|
{
|
|
|
|
private static ServerRepository _repository = null;
|
|
private static int _count = 0;
|
|
private static HashSet<ProcessRunner> _processes = new HashSet<ProcessRunner>();
|
|
private static HashMap<String, Boolean> _badServers = new HashMap<String, Boolean>();
|
|
|
|
private static SimpleDateFormat _dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
|
|
private static Logger _logger = Logger.getLogger("ServerMonitor");
|
|
|
|
private static int _totalPlayers = 0;
|
|
|
|
public static void main (String args[])
|
|
{
|
|
Region region = !new File("eu.dat").exists() ? Region.US : Region.EU;
|
|
_repository = ServerManager.getServerRepository(region); // Fetches and connects to server repo
|
|
|
|
File logFile = new File("monitor.log");
|
|
|
|
if (!logFile.exists())
|
|
{
|
|
try
|
|
{
|
|
logFile.createNewFile();
|
|
}
|
|
catch (IOException e1)
|
|
{
|
|
e1.printStackTrace();
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
FileHandler fileHandler = new FileHandler("monitor.log");
|
|
fileHandler.setFormatter(new CustomFormatter());
|
|
_logger.addHandler(fileHandler);
|
|
_logger.setUseParentHandlers(false);
|
|
}
|
|
catch (SecurityException | IOException e1)
|
|
{
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
HashMap<String, Entry<String, Long>> serverTracker = new HashMap<String, Entry<String, Long>>();
|
|
|
|
while (true)
|
|
{
|
|
_totalPlayers = 0;
|
|
Collection<ServerGroup> serverGroups = _repository.getServerGroups(true);
|
|
Collection<MinecraftServer> serverStatuses = _repository.getServerStatuses();
|
|
|
|
for (MinecraftServer deadServer : _repository.getDeadServers())
|
|
{
|
|
killServer(deadServer.getName(), deadServer.getPublicAddress(), "[KILLED] [DEAD] " + deadServer.getName() + ":" + deadServer.getPublicAddress(), true);
|
|
}
|
|
|
|
List<DedicatedServer> dedicatedServers = new ArrayList<DedicatedServer>(_repository.getDedicatedServers());
|
|
|
|
Map<String, ServerGroup> serverGroupMap = new HashMap<String, ServerGroup>();
|
|
for (ServerGroup serverGroup : serverGroups)
|
|
{
|
|
serverGroupMap.put(serverGroup.getName(), serverGroup);
|
|
_totalPlayers += serverGroup.getPlayerCount();
|
|
}
|
|
|
|
System.out.println("Total Players : " + _totalPlayers);
|
|
|
|
for (MinecraftServer minecraftServer : serverStatuses)
|
|
{
|
|
if (minecraftServer.getMotd().contains("Finished"))
|
|
{
|
|
killServer(minecraftServer.getName(), minecraftServer.getPublicAddress(), "[KILLED] [FINISHED] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress(), true);
|
|
continue;
|
|
}
|
|
|
|
for (DedicatedServer server : dedicatedServers)
|
|
{
|
|
if (serverGroupMap.containsKey(minecraftServer.getGroup()) && minecraftServer.getPublicAddress().equalsIgnoreCase(server.getPrivateAddress()))
|
|
{
|
|
ServerGroup serverGroup = serverGroupMap.get(minecraftServer.getGroup());
|
|
server.incrementServerCount(serverGroup);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (_count % 15 == 0)
|
|
{
|
|
_badServers.clear();
|
|
|
|
for (DedicatedServer serverData : dedicatedServers)
|
|
{
|
|
if (isServerOffline(serverData))
|
|
{
|
|
log("------=[OFFLINE]=------=[" + serverData.getName() + ":" + serverData.getPublicAddress() + "]=------=[OFFLINE]=------");
|
|
_badServers.put(serverData.getName(), true);
|
|
}
|
|
}
|
|
|
|
log(_badServers.size() + " bad servers.");
|
|
}
|
|
|
|
for (Iterator<DedicatedServer> iterator = dedicatedServers.iterator(); iterator.hasNext();)
|
|
{
|
|
DedicatedServer serverData = iterator.next();
|
|
|
|
if (_badServers.containsKey(serverData.getName()))
|
|
iterator.remove();
|
|
}
|
|
|
|
for (ServerGroup groupStatus : serverGroups)
|
|
{
|
|
NautHashMap<Integer, MinecraftServer> _serverMap = new NautHashMap<Integer, MinecraftServer>();
|
|
|
|
for (Iterator<MinecraftServer> serverIterator = groupStatus.getServers().iterator(); serverIterator.hasNext();)
|
|
{
|
|
MinecraftServer server = serverIterator.next();
|
|
int serverNum = Integer.parseInt(server.getName().split("-")[1]);
|
|
|
|
if (_serverMap.containsKey(serverNum))
|
|
{
|
|
killServer(server.getName(), server.getPublicAddress(), "[KILLED] [DUPLICATE] " + server.getName() + ":" + server.getPublicAddress(), true);
|
|
serverIterator.remove();
|
|
}
|
|
else
|
|
{
|
|
_serverMap.put(serverNum, server);
|
|
}
|
|
}
|
|
}
|
|
|
|
HashSet<String> onlineServers = new HashSet<String>();
|
|
|
|
for (MinecraftServer minecraftServer : serverStatuses)
|
|
{
|
|
onlineServers.add(minecraftServer.getName());
|
|
|
|
if (minecraftServer.getTps() <= 17)
|
|
log("[Performance] " + minecraftServer.getName() + ":" + minecraftServer.getPublicAddress() + "] Running poorly at " + minecraftServer.getTps() + " TPS");
|
|
}
|
|
|
|
for (Iterator<Entry<String, Entry<String, Long>>> iterator = serverTracker.entrySet().iterator(); iterator.hasNext();)
|
|
{
|
|
Entry<String, Entry<String, Long>> entry = iterator.next();
|
|
|
|
if (onlineServers.contains(entry.getKey()))
|
|
iterator.remove();
|
|
else if (System.currentTimeMillis() - entry.getValue().getValue() > 30000)
|
|
{
|
|
String serverName = entry.getKey();
|
|
String serverAddress = entry.getValue().getKey();
|
|
killServer(serverName, serverAddress, "[KILLED] [SLOW-STARTUP] " + serverName + ":" + serverAddress, true);
|
|
iterator.remove();
|
|
}
|
|
}
|
|
|
|
for (ServerGroup serverGroup : serverGroups)
|
|
{
|
|
handleGroupChanges(dedicatedServers, serverTracker, serverGroup, false);
|
|
}
|
|
|
|
int processWaits = 0;
|
|
|
|
while (_processes.size() > 0)
|
|
{
|
|
for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
|
|
{
|
|
ProcessRunner pr = iterator.next();
|
|
|
|
try
|
|
{
|
|
pr.join(100);
|
|
}
|
|
catch (InterruptedException e)
|
|
{
|
|
e.printStackTrace();
|
|
}
|
|
|
|
if (pr.isDone())
|
|
iterator.remove();
|
|
}
|
|
|
|
if (_processes.size() > 0)
|
|
{
|
|
try
|
|
{
|
|
log("Sleeping while processes run...");
|
|
Thread.sleep(6000);
|
|
}
|
|
catch (InterruptedException e)
|
|
{
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
if (processWaits >= 10)
|
|
{
|
|
log("Killing stale processes.");
|
|
|
|
for (Iterator<ProcessRunner> iterator = _processes.iterator(); iterator.hasNext();)
|
|
{
|
|
iterator.next().abort();
|
|
iterator.remove();
|
|
}
|
|
}
|
|
|
|
processWaits++;
|
|
}
|
|
|
|
processWaits = 0;
|
|
|
|
try
|
|
{
|
|
log("Natural sleep.");
|
|
Thread.sleep(10000);
|
|
}
|
|
catch (InterruptedException e)
|
|
{
|
|
e.printStackTrace();
|
|
}
|
|
|
|
_count++;
|
|
}
|
|
}
|
|
|
|
private static void handleGroupChanges(List<DedicatedServer> dedicatedServers, HashMap<String, Entry<String, Long>> serverTracker, ServerGroup serverGroup, boolean free)
|
|
{
|
|
int serverNum = 0;
|
|
int requiredTotal = serverGroup.getRequiredTotalServers();
|
|
int requiredJoinable = serverGroup.getRequiredJoinableServers();
|
|
int joinableServers = serverGroup.getJoinableCount();
|
|
int totalServers = serverGroup.getServerCount();
|
|
int serversToAdd = Math.max(requiredTotal - totalServers, requiredJoinable - joinableServers);
|
|
int serversToKill = (totalServers > requiredTotal && joinableServers > requiredJoinable) ? Math.min(joinableServers - requiredJoinable, serverGroup.getEmptyServers().size()) : 0;
|
|
|
|
// Minimum 1500 slot bufferzone
|
|
if (serverGroup.getName().equalsIgnoreCase("Lobby"))
|
|
{
|
|
int availableSlots = serverGroup.getMaxPlayerCount() - serverGroup.getPlayerCount();
|
|
|
|
if (availableSlots < 1500)
|
|
{
|
|
serversToAdd = Math.max(1, (1500 - availableSlots) / serverGroup.getMaxPlayers());
|
|
serversToKill = 0;
|
|
}
|
|
else if (serversToKill > 0)
|
|
serversToKill = Math.min(serversToKill, (availableSlots - 1500) / 80);
|
|
}
|
|
else if (serverGroup.getName().equalsIgnoreCase("Halloween"))
|
|
{
|
|
if (serverGroup.getServers().size() > 250)
|
|
{
|
|
serversToAdd = 0;
|
|
}
|
|
}
|
|
else if (serverGroup.getName().equalsIgnoreCase("UltraHardcore"))
|
|
{
|
|
int maxUHC = Math.max(1, _totalPlayers / 6000);
|
|
|
|
if (serversToAdd > 0)
|
|
serversToAdd = maxUHC - joinableServers;
|
|
|
|
if (joinableServers > maxUHC)
|
|
serversToKill = maxUHC - joinableServers;
|
|
}
|
|
|
|
while (serversToAdd > 0)
|
|
{
|
|
serverNum = serverGroup.generateUniqueId(serverNum + 1);
|
|
Collections.sort(dedicatedServers, new DedicatedServerSorter());
|
|
DedicatedServer bestServer = getBestDedicatedServer(dedicatedServers, serverGroup);
|
|
|
|
if (bestServer == null)
|
|
{
|
|
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!! NO DEDICATED SERVER AVAILABLE FOR GROUP " + serverGroup.getName() + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!");
|
|
break;
|
|
}
|
|
|
|
if (serverTracker.containsKey(serverGroup.getPrefix() + "-" + serverNum))
|
|
System.out.println("[WAITING] On " + serverGroup.getPrefix() + "-" + serverNum + " to finish starting...");
|
|
else
|
|
{
|
|
startServer(bestServer, serverGroup, serverNum, free);
|
|
serverTracker.put(serverGroup.getPrefix() + "-" + serverNum, new AbstractMap.SimpleEntry<String, Long>(bestServer.getPublicAddress(), System.currentTimeMillis()));
|
|
}
|
|
|
|
serversToAdd--;
|
|
}
|
|
|
|
while (serversToKill > 0)
|
|
{
|
|
List<MinecraftServer> emptyServers = new ArrayList<MinecraftServer>(serverGroup.getEmptyServers());
|
|
MinecraftServer emptyServer = emptyServers.get(serversToKill - 1);
|
|
killServer(emptyServer, "[KILLED] [EXCESS] " + emptyServer.getName() + ":" + emptyServer.getPublicAddress());
|
|
serversToKill--;
|
|
}
|
|
}
|
|
|
|
private static void killServer(final String serverName, final String serverAddress, final String message, final boolean announce)
|
|
{
|
|
String cmd = "/home/mineplex/easyRemoteKillServer.sh";
|
|
|
|
ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverName});
|
|
pr.start(new GenericRunnable<Boolean>()
|
|
{
|
|
public void run(Boolean error)
|
|
{
|
|
if (!error)
|
|
{
|
|
MinecraftServer server = _repository.getServerStatus(serverName);
|
|
|
|
if (server != null)
|
|
{
|
|
_repository.removeServerStatus(server);
|
|
}
|
|
}
|
|
|
|
if (announce)
|
|
{
|
|
if (error)
|
|
log("[" + serverName + ":" + serverAddress + "] Kill errored.");
|
|
else
|
|
log(message);
|
|
}
|
|
}
|
|
});
|
|
|
|
try
|
|
{
|
|
pr.join(500);
|
|
}
|
|
catch (InterruptedException e1)
|
|
{
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
|
|
if (!pr.isDone())
|
|
_processes.add(pr);
|
|
}
|
|
|
|
private static boolean isServerOffline(DedicatedServer serverData)
|
|
{
|
|
boolean success = false;
|
|
|
|
Process process = null;
|
|
String cmd = "/home/mineplex/isServerOnline.sh";
|
|
|
|
ProcessBuilder processBuilder = new ProcessBuilder(new String[] {"/bin/sh", cmd, serverData.getPublicAddress()});
|
|
|
|
try
|
|
{
|
|
process = processBuilder.start();
|
|
process.waitFor();
|
|
|
|
BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream()));
|
|
String line = reader.readLine();
|
|
|
|
while(line != null)
|
|
{
|
|
success = line.equals("Success");
|
|
|
|
line=reader.readLine();
|
|
}
|
|
}
|
|
catch (Exception e1)
|
|
{
|
|
e1.printStackTrace();
|
|
}
|
|
finally
|
|
{
|
|
process.destroy();
|
|
}
|
|
|
|
return !success;
|
|
}
|
|
|
|
private static DedicatedServer getBestDedicatedServer(Collection<DedicatedServer> dedicatedServers, ServerGroup serverGroup)
|
|
{
|
|
DedicatedServer bestServer = null;
|
|
|
|
for (DedicatedServer serverData : dedicatedServers)
|
|
{
|
|
if (serverData.getAvailableRam() > serverGroup.getRequiredRam()
|
|
&& serverData.getAvailableCpu() > serverGroup.getRequiredCpu())
|
|
{
|
|
if (bestServer == null || serverData.getServerCount(serverGroup) < bestServer.getServerCount(serverGroup))
|
|
{
|
|
bestServer = serverData;
|
|
}
|
|
}
|
|
}
|
|
|
|
return bestServer;
|
|
}
|
|
|
|
private static void killServer(final MinecraftServer serverToKill, String message)
|
|
{
|
|
killServer(serverToKill.getName(), serverToKill.getPublicAddress(), message, true);
|
|
}
|
|
|
|
private static void startServer(final DedicatedServer serverSpace, final ServerGroup serverGroup, final int serverNum, final boolean free)
|
|
{
|
|
String cmd = "/home/mineplex/easyRemoteStartServerCustom.sh";
|
|
final String groupPrefix = serverGroup.getPrefix();
|
|
final String serverName = serverSpace.getName();
|
|
final String serverAddress = serverSpace.getPublicAddress();
|
|
|
|
ProcessRunner pr = new ProcessRunner(new String[] {"/bin/sh", cmd, serverAddress, serverSpace.getPrivateAddress(), (serverGroup.getPortSection() + serverNum) + "", serverGroup.getRequiredRam() + "", serverGroup.getWorldZip(), serverGroup.getPlugin(), serverGroup.getConfigPath(), serverGroup.getName(), serverGroup.getPrefix() + "-" + serverNum, serverSpace.isUsRegion() ? "true" : "false", serverGroup.getAddNoCheat() + "" });
|
|
pr.start(new GenericRunnable<Boolean>()
|
|
{
|
|
public void run(Boolean error)
|
|
{
|
|
if (error)
|
|
log("[" + serverName + ":" + serverAddress + " Free Resources; CPU " + serverSpace.getAvailableCpu() + " RAM " + serverSpace.getAvailableRam() + "MB] Errored " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")");
|
|
else
|
|
log("[" + serverName + ":" + serverAddress + " Free Resources; CPU " + serverSpace.getAvailableCpu() + " RAM " + serverSpace.getAvailableRam() + "MB] Added " + serverName + "(" + groupPrefix+ "-" + serverNum + (free ? "-FREE" : "") + ")");
|
|
|
|
}
|
|
});
|
|
|
|
try
|
|
{
|
|
pr.join(500);
|
|
}
|
|
catch (InterruptedException e1)
|
|
{
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
serverSpace.incrementServerCount(serverGroup);
|
|
|
|
if (!pr.isDone())
|
|
_processes.add(pr);
|
|
}
|
|
|
|
private static void log(String message)
|
|
{
|
|
_logger.info("[" + _dateFormat.format(new Date()) + "] " + message);
|
|
System.out.println("[" + _dateFormat.format(new Date()) + "] " + message);
|
|
}
|
|
}
|