Made migration tool delete processed rows to increase efficiency
This commit is contained in:
parent
a35550867a
commit
4a4a935b9d
|
@ -8,14 +8,12 @@ import java.sql.PreparedStatement;
|
|||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import com.google.common.io.Files;
|
||||
import com.mineplex.statconverter.database.mysql.DBPool;
|
||||
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
|
||||
|
||||
public class Main
|
||||
{
|
||||
|
@ -51,12 +49,12 @@ public class Main
|
|||
|
||||
private File _info;
|
||||
private boolean _complete = false;
|
||||
private int _nextStart;
|
||||
private int _totalCompleted;
|
||||
private long _startTime;
|
||||
|
||||
public Main()
|
||||
{
|
||||
int start = 0;
|
||||
int completed = 0;
|
||||
try
|
||||
{
|
||||
if (new File(FILE_PATH_BASE + "complete.dat").exists())
|
||||
|
@ -69,10 +67,10 @@ public class Main
|
|||
|
||||
if (_info.exists())
|
||||
{
|
||||
String startStr = Files.readFirstLine(_info, Charset.defaultCharset());
|
||||
if (startStr != null && !startStr.isEmpty())
|
||||
String completedStr = Files.readFirstLine(_info, Charset.defaultCharset());
|
||||
if (completedStr != null && !completedStr.isEmpty())
|
||||
{
|
||||
start = Integer.parseInt(startStr);
|
||||
completed = Integer.parseInt(completedStr);
|
||||
}
|
||||
}
|
||||
else
|
||||
|
@ -81,11 +79,11 @@ public class Main
|
|||
Files.write(String.valueOf(0).getBytes(), _info);
|
||||
}
|
||||
|
||||
_nextStart = start;
|
||||
_totalCompleted = completed;
|
||||
|
||||
while (!_complete)
|
||||
{
|
||||
convertGroup(_nextStart);
|
||||
convertGroup();
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
|
@ -95,15 +93,15 @@ public class Main
|
|||
}
|
||||
}
|
||||
|
||||
private void convertGroup(int start)
|
||||
private void convertGroup()
|
||||
{
|
||||
_startTime = System.currentTimeMillis();
|
||||
System.out.println("[INFO] Starting " + start + " to " + (start + 9999));
|
||||
System.out.println("[INFO] Starting next batch of 10000 (Number " + (_totalCompleted + 1) + ")");
|
||||
Map<Integer[], Long> converting = new HashMap<>(10000);
|
||||
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
|
||||
{
|
||||
try (Statement s = c.createStatement();
|
||||
ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT " + start + ",10000;")
|
||||
ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT 10000;")
|
||||
)
|
||||
{
|
||||
while (rs.next())
|
||||
|
@ -127,6 +125,7 @@ public class Main
|
|||
PreparedStatement select = c.prepareStatement("SELECT value FROM accountStatsAllTime WHERE accountId=? AND statId=?;");
|
||||
PreparedStatement update = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
|
||||
PreparedStatement insert = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
|
||||
PreparedStatement delete = c.prepareStatement("DELETE FROM accountStat WHERE accountId=? AND statId=?;");
|
||||
)
|
||||
{
|
||||
for (Entry<Integer[], Long> convertEntry : converting.entrySet())
|
||||
|
@ -157,7 +156,7 @@ public class Main
|
|||
insert.execute();
|
||||
insert.clearParameters();
|
||||
}
|
||||
else if (fetched >= 2 * value)
|
||||
else if (value > 0 && fetched >= 2 * value)
|
||||
{
|
||||
update.setLong(1, -1 * value);
|
||||
update.setInt(2, accountId);
|
||||
|
@ -173,9 +172,14 @@ public class Main
|
|||
update.execute();
|
||||
update.clearParameters();
|
||||
}
|
||||
|
||||
delete.setInt(1, accountId);
|
||||
delete.setInt(2, statId);
|
||||
delete.addBatch();
|
||||
}
|
||||
delete.executeBatch();
|
||||
}
|
||||
completeGroup(start);
|
||||
completeGroup();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -198,18 +202,17 @@ public class Main
|
|||
}
|
||||
}
|
||||
|
||||
private void completeGroup(int start)
|
||||
private void completeGroup()
|
||||
{
|
||||
long timeTaken = System.currentTimeMillis() - _startTime;
|
||||
_startTime = 0;
|
||||
_nextStart = start + 10000;
|
||||
if (_info.delete())
|
||||
{
|
||||
try
|
||||
{
|
||||
_info.createNewFile();
|
||||
Files.write(String.valueOf(_nextStart).getBytes(), _info);
|
||||
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds");
|
||||
Files.write(String.valueOf(_totalCompleted + 1).getBytes(), _info);
|
||||
System.out.println("[INFO] Completed group " + _totalCompleted++ + " of 10000 in " + timeTaken + " milliseconds");
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
|
|
Loading…
Reference in New Issue