JarvisBigBalls
/
Mineplex2018-withcommit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Made migration tool delete processed rows to increase efficiency

This commit is contained in:
AlexTheCoder 2018-01-30 00:55:00 -05:00 committed by Alexander Meech
parent a35550867a
commit 4a4a935b9d
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Tools/Stat Conversion/src/com/mineplex/statconverter/Main.java
View File

@ -8,14 +8,12 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import com.google.common.io.Files; import com.google.common.io.Files;
import com.mineplex.statconverter.database.mysql.DBPool; import com.mineplex.statconverter.database.mysql.DBPool;
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
public class Main public class Main
{ {
@ -51,12 +49,12 @@ public class Main
private File _info; private File _info;
private boolean _complete = false; private boolean _complete = false;
private int _nextStart; private int _totalCompleted;
private long _startTime; private long _startTime;
public Main() public Main()
{ {
int start = 0; int completed = 0;
try try
{ {
if (new File(FILE_PATH_BASE + "complete.dat").exists()) if (new File(FILE_PATH_BASE + "complete.dat").exists())
@ -69,10 +67,10 @@ public class Main
if (_info.exists()) if (_info.exists())
{ {
String startStr = Files.readFirstLine(_info, Charset.defaultCharset()); String completedStr = Files.readFirstLine(_info, Charset.defaultCharset());
if (startStr != null && !startStr.isEmpty()) if (completedStr != null && !completedStr.isEmpty())
{ {
start = Integer.parseInt(startStr); completed = Integer.parseInt(completedStr);
} }
} }
else else
@ -81,11 +79,11 @@ public class Main
Files.write(String.valueOf(0).getBytes(), _info); Files.write(String.valueOf(0).getBytes(), _info);
} }
_nextStart = start; _totalCompleted = completed;
while (!_complete) while (!_complete)
{ {
convertGroup(_nextStart); convertGroup();
} }
} }
catch (IOException e) catch (IOException e)
@ -95,15 +93,15 @@ public class Main
} }
} }
private void convertGroup(int start) private void convertGroup()
{ {
_startTime = System.currentTimeMillis(); _startTime = System.currentTimeMillis();
System.out.println("[INFO] Starting " + start + " to " + (start + 9999)); System.out.println("[INFO] Starting next batch of 10000 (Number " + (_totalCompleted + 1) + ")");
Map<Integer[], Long> converting = new HashMap<>(10000); Map<Integer[], Long> converting = new HashMap<>(10000);
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection()) try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
{ {
try (Statement s = c.createStatement(); try (Statement s = c.createStatement();
ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT " + start + ",10000;") ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT 10000;")
) )
{ {
while (rs.next()) while (rs.next())
@ -127,6 +125,7 @@ public class Main
PreparedStatement select = c.prepareStatement("SELECT value FROM accountStatsAllTime WHERE accountId=? AND statId=?;"); PreparedStatement select = c.prepareStatement("SELECT value FROM accountStatsAllTime WHERE accountId=? AND statId=?;");
PreparedStatement update = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;"); PreparedStatement update = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
PreparedStatement insert = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);"); PreparedStatement insert = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
PreparedStatement delete = c.prepareStatement("DELETE FROM accountStat WHERE accountId=? AND statId=?;");
) )
{ {
for (Entry<Integer[], Long> convertEntry : converting.entrySet()) for (Entry<Integer[], Long> convertEntry : converting.entrySet())
@ -157,7 +156,7 @@ public class Main
insert.execute(); insert.execute();
insert.clearParameters(); insert.clearParameters();
} }
else if (fetched >= 2 * value) else if (value > 0 && fetched >= 2 * value)
{ {
update.setLong(1, -1 * value); update.setLong(1, -1 * value);
update.setInt(2, accountId); update.setInt(2, accountId);
@ -173,9 +172,14 @@ public class Main
update.execute(); update.execute();
update.clearParameters(); update.clearParameters();
} }
delete.setInt(1, accountId);
delete.setInt(2, statId);
delete.addBatch();
} }
delete.executeBatch();
} }
completeGroup(start); completeGroup();
} }
else else
{ {
@ -198,18 +202,17 @@ public class Main
} }
} }
private void completeGroup(int start) private void completeGroup()
{ {
long timeTaken = System.currentTimeMillis() - _startTime; long timeTaken = System.currentTimeMillis() - _startTime;
_startTime = 0; _startTime = 0;
_nextStart = start + 10000;
if (_info.delete()) if (_info.delete())
{ {
try try
{ {
_info.createNewFile(); _info.createNewFile();
Files.write(String.valueOf(_nextStart).getBytes(), _info); Files.write(String.valueOf(_totalCompleted + 1).getBytes(), _info);
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds"); System.out.println("[INFO] Completed group " + _totalCompleted++ + " of 10000 in " + timeTaken + " milliseconds");
} }
catch (IOException e) catch (IOException e)
{ {