Vastly speed up stat converter
This commit is contained in:
parent
fc5d7754bf
commit
e09f7fcb5d
|
@ -2,6 +2,7 @@ package com.mineplex.statconverter;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
|
@ -13,16 +14,33 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import com.google.common.io.Files;
|
||||
import com.mineplex.statconverter.database.mysql.DBPool;
|
||||
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
|
||||
|
||||
public class Main
|
||||
{
|
||||
public static void main(String[] args)
|
||||
private static final String FILE_PATH_BASE;
|
||||
|
||||
static
|
||||
{
|
||||
String path;
|
||||
try
|
||||
{
|
||||
if (new File(new File(".").getCanonicalPath() + File.separator + "doBuckets.dat").exists())
|
||||
path = new File(".").getCanonicalPath() + File.separator;
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
ex.printStackTrace();
|
||||
path = "";
|
||||
}
|
||||
|
||||
FILE_PATH_BASE = path;
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
if (new File(FILE_PATH_BASE + "doBuckets.dat").exists())
|
||||
{
|
||||
new BucketFiller();
|
||||
}
|
||||
|
@ -31,26 +49,44 @@ public class Main
|
|||
new Main();
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
new Main();
|
||||
}
|
||||
}
|
||||
|
||||
private File _info;
|
||||
private boolean _complete = false;
|
||||
private int _nextStart;
|
||||
private long _startTime;
|
||||
|
||||
public Main()
|
||||
{
|
||||
int start = 0;
|
||||
try
|
||||
{
|
||||
if (new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").exists())
|
||||
if (new File(FILE_PATH_BASE + "complete.dat").exists())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_info = new File(FILE_PATH_BASE + "converterInfo.dat");
|
||||
System.out.println(FILE_PATH_BASE + "converterInfo.dat");
|
||||
|
||||
if (_info.exists())
|
||||
{
|
||||
String startStr = Files.readFirstLine(_info, Charset.defaultCharset());
|
||||
if (startStr != null && !startStr.isEmpty())
|
||||
{
|
||||
start = Integer.parseInt(startStr);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_info.createNewFile();
|
||||
Files.write(String.valueOf(0).getBytes(), _info);
|
||||
}
|
||||
|
||||
_nextStart = start;
|
||||
|
||||
while (!_complete)
|
||||
{
|
||||
convertGroup();
|
||||
convertGroup(_nextStart);
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
|
@ -60,14 +96,15 @@ public class Main
|
|||
}
|
||||
}
|
||||
|
||||
private void convertGroup()
|
||||
private void convertGroup(int start)
|
||||
{
|
||||
System.out.println("[INFO] Starting next 10000");
|
||||
_startTime = System.currentTimeMillis();
|
||||
System.out.println("[INFO] Starting " + start + " to " + (start + 9999));
|
||||
List<Integer> accounts = new ArrayList<>(10000);
|
||||
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
|
||||
{
|
||||
try (Statement s = c.createStatement();
|
||||
ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT 10000;")
|
||||
ResultSet rs = s.executeQuery("SELECT DISTINCT accountId FROM accountStat LIMIT " + start + ",10000;")
|
||||
)
|
||||
{
|
||||
while (rs.next())
|
||||
|
@ -132,7 +169,7 @@ public class Main
|
|||
|
||||
try (PreparedStatement updateStatement = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
|
||||
PreparedStatement insertStatement = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
|
||||
PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;");
|
||||
//PreparedStatement deleteStatement = c.prepareStatement("DELETE FROM accountStat WHERE accountId=?;");
|
||||
)
|
||||
{
|
||||
int updates = 0;
|
||||
|
@ -181,8 +218,8 @@ public class Main
|
|||
insertStatement.executeBatch();
|
||||
}
|
||||
|
||||
deleteStatement.setInt(1, accountId);
|
||||
deleteStatement.execute();
|
||||
//deleteStatement.setInt(1, accountId);
|
||||
//deleteStatement.execute();
|
||||
}
|
||||
}
|
||||
try
|
||||
|
@ -194,14 +231,14 @@ public class Main
|
|||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println("[INFO] Completed group of 10000");
|
||||
completeGroup(start);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println("[INFO] Conversion complete");
|
||||
try
|
||||
{
|
||||
new File(new File(".").getCanonicalPath() + File.separator + "complete.dat").createNewFile();
|
||||
new File(FILE_PATH_BASE + "complete.dat").createNewFile();
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
|
@ -216,4 +253,29 @@ public class Main
|
|||
_complete = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void completeGroup(int start)
|
||||
{
|
||||
long timeTaken = System.currentTimeMillis() - _startTime;
|
||||
_startTime = 0;
|
||||
_nextStart = start + 10000;
|
||||
if (_info.delete())
|
||||
{
|
||||
try
|
||||
{
|
||||
_info.createNewFile();
|
||||
Files.write(String.valueOf(_nextStart).getBytes(), _info);
|
||||
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds");
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
_complete = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_complete = true;
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue