JarvisBigBalls
/
Mineplex2018-withcommit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Vastly speed up stat converter

This commit is contained in:
AlexTheCoder 2017-12-19 00:39:04 -05:00 committed by Alexander Meech
parent fc5d7754bf
commit e09f7fcb5d
1 changed files with 82 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
Tools/Stat Conversion/src/com/mineplex/statconverter/Main.java
View File

@ -2,6 +2,7 @@ package com.mineplex.statconverter;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@ -13,44 +14,79 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import com.google.common.io.Files;
import com.mineplex.statconverter.database.mysql.DBPool;
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
public class Main
{
public static void main(String[] args)
private static final String FILE_PATH_BASE;
static
{
String path;
try
{
if (new File(new File(".").getCanonicalPath() + File.separator + "doBuckets.dat").exists())
{
new BucketFiller();
}
else
{
new Main();
}
path = new File(".").getCanonicalPath() + File.separator;
}
catch (IOException e)
catch (IOException ex)
{
ex.printStackTrace();
path = "";
}
FILE_PATH_BASE = path;
}
public static void main(String[] args)
{
if (new File(FILE_PATH_BASE + "doBuckets.dat").exists())
{
new BucketFiller();
}
else
{
new Main();
}
}
private File _info;
private boolean _complete = false;
private int _nextStart;
private long _startTime;
public Main()
{
int start = 0;
try
{
if (new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").exists())
if (new File(FILE_PATH_BASE + "complete.dat").exists())
{
return;
}
_info = new File(FILE_PATH_BASE + "converterInfo.dat");
System.out.println(FILE_PATH_BASE + "converterInfo.dat");
if (_info.exists())
{
String startStr = Files.readFirstLine(_info, Charset.defaultCharset());
if (startStr != null && !startStr.isEmpty())
{
start = Integer.parseInt(startStr);
}
}
else
{
_info.createNewFile();
Files.write(String.valueOf(0).getBytes(), _info);
}
_nextStart = start;
while (!_complete)
{
convertGroup();
convertGroup(_nextStart);
}
}
catch (IOException e)
@ -60,14 +96,15 @@ public class Main
}
}
private void convertGroup()
private void convertGroup(int start)
{
System.out.println("[INFO] Starting next 10000");
_startTime = System.currentTimeMillis();
System.out.println("[INFO] Starting " + start + " to " + (start + 9999));
List<Integer> accounts = new ArrayList<>(10000);
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
{
try (Statement s = c.createStatement();
ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT 10000;")
ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT " + start + ",10000;")
)
{
while (rs.next())
@ -132,7 +169,7 @@ public class Main
try (PreparedStatement updateStatement = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
PreparedStatement insertStatement = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;");
//PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;");
)
{
int updates = 0;
@ -181,8 +218,8 @@ public class Main
insertStatement.executeBatch();
}
deleteStatement.setInt(1, accountId);
deleteStatement.execute();
//deleteStatement.setInt(1, accountId);
//deleteStatement.execute();
}
}
try
@ -194,14 +231,14 @@ public class Main
e.printStackTrace();
}
}
System.out.println("[INFO] Completed group of 10000");
completeGroup(start);
}
else
{
System.out.println("[INFO] Conversion complete");
try
{
new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").createNewFile();
new File(FILE_PATH_BASE + "complete.dat").createNewFile();
}
catch (IOException e)
{
@ -216,4 +253,29 @@ public class Main
_complete = true;
}
}
private void completeGroup(int start)
{
long timeTaken = System.currentTimeMillis() - _startTime;
_startTime = 0;
_nextStart = start + 10000;
if (_info.delete())
{
try
{
_info.createNewFile();
Files.write(String.valueOf(_nextStart).getBytes(), _info);
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds");
}
catch (IOException e)
{
e.printStackTrace();
_complete = true;
}
}
else
{
_complete = true;
}
}
}