JarvisBigBalls/Mineplex2018-withcommit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Vastly speed up stat converter

Browse Source
This commit is contained in:
AlexTheCoder 2017-12-19 00:39:04 -05:00 committed by Alexander Meech
parent fc5d7754bf
commit e09f7fcb5d
1 changed files with 82 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Tools/Stat Conversion/src/com/mineplex/statconverter/Main.java
View File

@ -2,6 +2,7 @@ package com.mineplex.statconverter;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
@ -13,16 +14,33 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import com.google.common.io.Files;
import com.mineplex.statconverter.database.mysql.DBPool; import com.mineplex.statconverter.database.mysql.DBPool;
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException; import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
public class Main public class Main
{ {
public static void main(String[] args) private static final String FILE_PATH_BASE;
static
{ {
String path;
try try
{ {
if (new File(new File(".").getCanonicalPath() + File.separator + "doBuckets.dat").exists()) path = new File(".").getCanonicalPath() + File.separator;
}
catch (IOException ex)
{
ex.printStackTrace();
path = "";
}
FILE_PATH_BASE = path;
}
public static void main(String[] args)
{
if (new File(FILE_PATH_BASE + "doBuckets.dat").exists())
{ {
new BucketFiller(); new BucketFiller();
} }
@ -31,26 +49,44 @@ public class Main
new Main(); new Main();
} }
} }
catch (IOException e)
{
new Main();
}
}
private File _info;
private boolean _complete = false; private boolean _complete = false;
private int _nextStart;
private long _startTime;
public Main() public Main()
{ {
int start = 0;
try try
{ {
if (new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").exists()) if (new File(FILE_PATH_BASE + "complete.dat").exists())
{ {
return; return;
} }
_info = new File(FILE_PATH_BASE + "converterInfo.dat");
System.out.println(FILE_PATH_BASE + "converterInfo.dat");
if (_info.exists())
{
String startStr = Files.readFirstLine(_info, Charset.defaultCharset());
if (startStr != null && !startStr.isEmpty())
{
start = Integer.parseInt(startStr);
}
}
else
{
_info.createNewFile();
Files.write(String.valueOf(0).getBytes(), _info);
}
_nextStart = start;
while (!_complete) while (!_complete)
{ {
convertGroup(); convertGroup(_nextStart);
} }
} }
catch (IOException e) catch (IOException e)
@ -60,14 +96,15 @@ public class Main
} }
} }
private void convertGroup() private void convertGroup(int start)
{ {
System.out.println("[INFO] Starting next 10000"); _startTime = System.currentTimeMillis();
System.out.println("[INFO] Starting " + start + " to " + (start + 9999));
List<Integer> accounts = new ArrayList<>(10000); List<Integer> accounts = new ArrayList<>(10000);
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection()) try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
{ {
try (Statement s = c.createStatement(); try (Statement s = c.createStatement();
ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT 10000;") ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT " + start + ",10000;")
) )
{ {
while (rs.next()) while (rs.next())
@ -132,7 +169,7 @@ public class Main
try (PreparedStatement updateStatement = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;"); try (PreparedStatement updateStatement = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
PreparedStatement insertStatement = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);"); PreparedStatement insertStatement = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;"); //PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;");
) )
{ {
int updates = 0; int updates = 0;
@ -181,8 +218,8 @@ public class Main
insertStatement.executeBatch(); insertStatement.executeBatch();
} }
deleteStatement.setInt(1, accountId); //deleteStatement.setInt(1, accountId);
deleteStatement.execute(); //deleteStatement.execute();
} }
} }
try try
@ -194,14 +231,14 @@ public class Main
e.printStackTrace(); e.printStackTrace();
} }
} }
System.out.println("[INFO] Completed group of 10000"); completeGroup(start);
} }
else else
{ {
System.out.println("[INFO] Conversion complete"); System.out.println("[INFO] Conversion complete");
try try
{ {
new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").createNewFile(); new File(FILE_PATH_BASE + "complete.dat").createNewFile();
} }
catch (IOException e) catch (IOException e)
{ {
@ -216,4 +253,29 @@ public class Main
_complete = true; _complete = true;
} }
} }
private void completeGroup(int start)
{
long timeTaken = System.currentTimeMillis() - _startTime;
_startTime = 0;
_nextStart = start + 10000;
if (_info.delete())
{
try
{
_info.createNewFile();
Files.write(String.valueOf(_nextStart).getBytes(), _info);
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds");
}
catch (IOException e)
{
e.printStackTrace();
_complete = true;
}
}
else
{
_complete = true;
}
}
} }