Made migration tool delete processed rows to increase efficiency
This commit is contained in:
parent
a35550867a
commit
4a4a935b9d
|
@ -8,14 +8,12 @@ import java.sql.PreparedStatement;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
import com.google.common.io.Files;
|
import com.google.common.io.Files;
|
||||||
import com.mineplex.statconverter.database.mysql.DBPool;
|
import com.mineplex.statconverter.database.mysql.DBPool;
|
||||||
import com.mysql.jdbc.exceptions.jdbc4.MySQLDataException;
|
|
||||||
|
|
||||||
public class Main
|
public class Main
|
||||||
{
|
{
|
||||||
|
@ -51,12 +49,12 @@ public class Main
|
||||||
|
|
||||||
private File _info;
|
private File _info;
|
||||||
private boolean _complete = false;
|
private boolean _complete = false;
|
||||||
private int _nextStart;
|
private int _totalCompleted;
|
||||||
private long _startTime;
|
private long _startTime;
|
||||||
|
|
||||||
public Main()
|
public Main()
|
||||||
{
|
{
|
||||||
int start = 0;
|
int completed = 0;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (new File(FILE_PATH_BASE + "complete.dat").exists())
|
if (new File(FILE_PATH_BASE + "complete.dat").exists())
|
||||||
|
@ -69,10 +67,10 @@ public class Main
|
||||||
|
|
||||||
if (_info.exists())
|
if (_info.exists())
|
||||||
{
|
{
|
||||||
String startStr = Files.readFirstLine(_info, Charset.defaultCharset());
|
String completedStr = Files.readFirstLine(_info, Charset.defaultCharset());
|
||||||
if (startStr != null && !startStr.isEmpty())
|
if (completedStr != null && !completedStr.isEmpty())
|
||||||
{
|
{
|
||||||
start = Integer.parseInt(startStr);
|
completed = Integer.parseInt(completedStr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -81,11 +79,11 @@ public class Main
|
||||||
Files.write(String.valueOf(0).getBytes(), _info);
|
Files.write(String.valueOf(0).getBytes(), _info);
|
||||||
}
|
}
|
||||||
|
|
||||||
_nextStart = start;
|
_totalCompleted = completed;
|
||||||
|
|
||||||
while (!_complete)
|
while (!_complete)
|
||||||
{
|
{
|
||||||
convertGroup(_nextStart);
|
convertGroup();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (IOException e)
|
catch (IOException e)
|
||||||
|
@ -95,15 +93,15 @@ public class Main
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertGroup(int start)
|
private void convertGroup()
|
||||||
{
|
{
|
||||||
_startTime = System.currentTimeMillis();
|
_startTime = System.currentTimeMillis();
|
||||||
System.out.println("[INFO] Starting " + start + " to " + (start + 9999));
|
System.out.println("[INFO] Starting next batch of 10000 (Number " + (_totalCompleted + 1) + ")");
|
||||||
Map<Integer[], Long> converting = new HashMap<>(10000);
|
Map<Integer[], Long> converting = new HashMap<>(10000);
|
||||||
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
|
try (Connection c = DBPool.getDataSource("ACCOUNT_PC").getConnection())
|
||||||
{
|
{
|
||||||
try (Statement s = c.createStatement();
|
try (Statement s = c.createStatement();
|
||||||
ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT " + start + ",10000;")
|
ResultSet rs = s.executeQuery("SELECT accountId, statId, value FROM accountStat LIMIT 10000;")
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
while (rs.next())
|
while (rs.next())
|
||||||
|
@ -127,6 +125,7 @@ public class Main
|
||||||
PreparedStatement select = c.prepareStatement("SELECT value FROM accountStatsAllTime WHERE accountId=? AND statId=?;");
|
PreparedStatement select = c.prepareStatement("SELECT value FROM accountStatsAllTime WHERE accountId=? AND statId=?;");
|
||||||
PreparedStatement update = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
|
PreparedStatement update = c.prepareStatement("UPDATE accountStatsAllTime SET value=value+? WHERE accountId=? AND statId=?;");
|
||||||
PreparedStatement insert = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
|
PreparedStatement insert = c.prepareStatement("INSERT INTO accountStatsAllTime (accountId, statId, value) VALUES (?, ?, ?);");
|
||||||
|
PreparedStatement delete = c.prepareStatement("DELETE FROM accountStat WHERE accountId=? AND statId=?;");
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
for (Entry<Integer[], Long> convertEntry : converting.entrySet())
|
for (Entry<Integer[], Long> convertEntry : converting.entrySet())
|
||||||
|
@ -157,7 +156,7 @@ public class Main
|
||||||
insert.execute();
|
insert.execute();
|
||||||
insert.clearParameters();
|
insert.clearParameters();
|
||||||
}
|
}
|
||||||
else if (fetched >= 2 * value)
|
else if (value > 0 && fetched >= 2 * value)
|
||||||
{
|
{
|
||||||
update.setLong(1, -1 * value);
|
update.setLong(1, -1 * value);
|
||||||
update.setInt(2, accountId);
|
update.setInt(2, accountId);
|
||||||
|
@ -173,9 +172,14 @@ public class Main
|
||||||
update.execute();
|
update.execute();
|
||||||
update.clearParameters();
|
update.clearParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
delete.setInt(1, accountId);
|
||||||
|
delete.setInt(2, statId);
|
||||||
|
delete.addBatch();
|
||||||
}
|
}
|
||||||
|
delete.executeBatch();
|
||||||
}
|
}
|
||||||
completeGroup(start);
|
completeGroup();
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -198,18 +202,17 @@ public class Main
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void completeGroup(int start)
|
private void completeGroup()
|
||||||
{
|
{
|
||||||
long timeTaken = System.currentTimeMillis() - _startTime;
|
long timeTaken = System.currentTimeMillis() - _startTime;
|
||||||
_startTime = 0;
|
_startTime = 0;
|
||||||
_nextStart = start + 10000;
|
|
||||||
if (_info.delete())
|
if (_info.delete())
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_info.createNewFile();
|
_info.createNewFile();
|
||||||
Files.write(String.valueOf(_nextStart).getBytes(), _info);
|
Files.write(String.valueOf(_totalCompleted + 1).getBytes(), _info);
|
||||||
System.out.println("[INFO] Completed " + start + " to " + (_nextStart - 1) + " in " + timeTaken + " milliseconds");
|
System.out.println("[INFO] Completed group " + _totalCompleted++ + " of 10000 in " + timeTaken + " milliseconds");
|
||||||
}
|
}
|
||||||
catch (IOException e)
|
catch (IOException e)
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue