Brandon/FastAsyncWorldedit
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Begin rollback optimizations + other

Browse Source 
Store rollback summary in database (option)
API improvements
Load before AWE
This commit is contained in:
Jesse Boyd 2016-08-03 14:43:27 +10:00
parent 39acae08aa
commit 1ca5798e9d
18 changed files with 482 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bukkit110/src/main/resources/plugin.yml
View File

@ -3,7 +3,7 @@ main: com.boydti.fawe.bukkit.v1_10.BukkitMain_110
version: ${version}
description: Fast Async WorldEdit plugin
authors: [Empire92]
loadbefore: [WorldEdit]
loadbefore: [WorldEdit,AsyncWorldEdit]
load: STARTUP
database: false
#softdepend: [WorldGuard, PlotSquared, MCore, Factions, GriefPrevention, Residence, Towny, PlotMe, PreciousStones]

2
bukkit1710/src/main/resources/plugin.yml
View File

@ -3,7 +3,7 @@ main: com.boydti.fawe.bukkit.v1_7.BukkitMain_17
version: ${version}
description: Fast Async WorldEdit plugin
authors: [Empire92]
loadbefore: [WorldEdit]
loadbefore: [WorldEdit,AsyncWorldEdit]
load: STARTUP
database: false
#softdepend: [WorldGuard, PlotSquared, MCore, Factions, GriefPrevention, Residence, Towny, PlotMe, PreciousStones]

2
bukkit18/src/main/resources/plugin.yml
View File

@ -3,7 +3,7 @@ main: com.boydti.fawe.bukkit.v1_8.BukkitMain_18
version: ${version}
description: Fast Async WorldEdit plugin
authors: [Empire92]
loadbefore: [WorldEdit]
loadbefore: [WorldEdit,AsyncWorldEdit]
load: STARTUP
database: false
#softdepend: [WorldGuard, PlotSquared, MCore, Factions, GriefPrevention, Residence, Towny, PlotMe, PreciousStones]

2
bukkit19/src/main/resources/plugin.yml
View File

@ -3,7 +3,7 @@ main: com.boydti.fawe.bukkit.v1_9.BukkitMain_19
version: ${version}
description: Fast Async WorldEdit plugin
authors: [Empire92]
loadbefore: [WorldEdit]
loadbefore: [WorldEdit,AsyncWorldEdit]
load: STARTUP
database: false
#softdepend: [WorldGuard, PlotSquared, MCore, Factions, GriefPrevention, Residence, Towny, PlotMe, PreciousStones]

5
core/src/main/java/com/boydti/fawe/config/Settings.java
View File

@ -77,6 +77,11 @@ public class Settings extends Config {
" - Enables the rollback command"
})
public static boolean USE_DISK = false;
@Comment({
"Use a database to store disk storage summaries:",
" - Faster lookups and rollback from disk",
})
public static boolean USE_DATABASE = false;
@Comment({
"Record history with dispatching:",
" - Faster as it avoids duplicate block checks",

23
core/src/main/java/com/boydti/fawe/database/DBHandler.java Normal file
View File

@ -0,0 +1,23 @@
package com.boydti.fawe.database;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class DBHandler {
private Map<String, RollbackDatabase> databases = new ConcurrentHashMap<>();
public RollbackDatabase getDatabase(String world) {
RollbackDatabase database = databases.get(world);
if (database != null) {
return database;
}
try {
database = new RollbackDatabase(world);
databases.put(world, database);
return database;
} catch (Throwable e) {
e.printStackTrace();
return null;
}
}
}

286
core/src/main/java/com/boydti/fawe/database/RollbackDatabase.java Normal file
View File

@ -0,0 +1,286 @@
package com.boydti.fawe.database;
import com.boydti.fawe.Fawe;
import com.boydti.fawe.FaweAPI;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.logging.rollback.RollbackOptimizedHistory;
import com.boydti.fawe.object.RunnableVal;
import com.boydti.fawe.object.change.MutablePlayerBlockChange;
import com.boydti.fawe.object.changeset.DiskStorageHistory;
import com.boydti.fawe.util.TaskManager;
import com.sk89q.worldedit.world.World;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
public class RollbackDatabase {
private final String prefix;
private final File dbLocation;
private final String world;
private Connection connection;
private String INSERT_EDIT;
private String CREATE_TABLE;
private String GET_EDITS_POINT;
private String GET_EDITS;
private String PURGE;
private ConcurrentLinkedQueue<RollbackOptimizedHistory> historyChanges = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Runnable> notify = new ConcurrentLinkedQueue<>();
public RollbackDatabase(final String world) throws SQLException, ClassNotFoundException {
this.prefix = "";
this.world = world;
this.dbLocation = new File(Fawe.imp().getDirectory(), "history" + File.separator + world);
connection = openConnection();
CREATE_TABLE = "CREATE TABLE IF NOT EXISTS `" + prefix + "edits` (`player` BLOB(16) NOT NULL,`id` INT NOT NULL,`x1` INT NOT NULL,`y1` INT NOT NULL,`z1` INT NOT NULL,`x2` INT NOT NULL,`y2` INT NOT NULL,`z2` INT NOT NULL,`time` INT NOT NULL, PRIMARY KEY (player, id))";
INSERT_EDIT = "INSERT INTO `" + prefix + "edits` (`player`,`id`,`x1`,`y1`,`z1`,`x2`,`y2`,`z2`,`time`) VALUES(?,?,?,?,?,?,?,?,?)";
PURGE = "DELETE FROM `" + prefix + "edits` WHERE `time`<?";
GET_EDITS = "SELECT `player`,`id`,`x1`,`y1`,`z1`,`x2`,`y2`,`z2`,`time` WHERE `x2`>=? AND `x1`<=? AND `y2`>=? AND `y1`<=? AND `z2`>=? AND `z1`<=? AND `time`>?";
GET_EDITS_POINT = "SELECT `player`,`id`,`time` WHERE `x2`>=? AND `x1`<=? AND `y2`>=? AND `y1`<=? AND `z2`>=? AND `z1`<=?";
purge((int) TimeUnit.DAYS.toMillis(Settings.HISTORY.DELETE_AFTER_DAYS));
TaskManager.IMP.async(new Runnable() {
@Override
public void run() {
long last = System.currentTimeMillis();
while (true) {
if (connection == null) {
break;
}
if (!RollbackDatabase.this.sendBatch()) {
try {
while (!notify.isEmpty()) {
Runnable runnable = notify.poll();
runnable.run();
}
Thread.sleep(50);
} catch (final InterruptedException e) {
e.printStackTrace();
}
}
}
}
});
}
public void addFinishTask(Runnable run) {
notify.add(run);
}
public void purge(int diff) {
long now = System.currentTimeMillis() / 1000;
final int then = (int) (now - diff);
addTask(new Runnable() {
@Override
public void run() {
try (PreparedStatement stmt = connection.prepareStatement(PURGE)) {
stmt.setInt(1, then);
stmt.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
}
}
});
}
public void getPotentialEdits(final int x, final int y, final int z, final RunnableVal<DiskStorageHistory> onEach, final Runnable onFail) {
final World world = FaweAPI.getWorld(this.world);
addTask(new Runnable() {
@Override
public void run() {
try (PreparedStatement stmt = connection.prepareStatement(GET_EDITS_POINT)) {
stmt.setInt(1, x);
stmt.setInt(2, x);
stmt.setInt(3, y);
stmt.setInt(4, y);
stmt.setInt(5, z);
stmt.setInt(6, z);
ResultSet result = stmt.executeQuery();
if (!result.next()) {
TaskManager.IMP.taskNow(onFail, false);
return;
}
do {
byte[] uuid = result.getBytes(1);
int index = result.getInt(2);
long time = 1000l * result.getInt(3);
DiskStorageHistory history = new DiskStorageHistory(world, UUID.nameUUIDFromBytes(uuid), index);
if (history.getBDFile().exists()) {
onEach.run(history);
}
} while (result.next());
} catch (SQLException e) {
e.printStackTrace();
}
}
});
}
public int getBlocks(int originX, int originZ, int radius, UUID uuid, long timeDiff, RunnableVal<MutablePlayerBlockChange> result) {
return 0;
}
public void logEdit(RollbackOptimizedHistory history) {
historyChanges.add(history);
}
private final ConcurrentLinkedQueue<Runnable> tasks = new ConcurrentLinkedQueue<>();
public void addTask(Runnable run) {
this.tasks.add(run);
}
public void runTasks() {
Runnable task;
while ((task = tasks.poll()) != null) {
try {
task.run();
} catch (Exception e) {
e.printStackTrace();
}
}
}
private boolean sendBatch() {
try {
runTasks();
commit();
if (connection.getAutoCommit()) {
connection.setAutoCommit(false);
}
int size = Math.min(1048572, historyChanges.size());
if (size == 0) {
return false;
}
RollbackOptimizedHistory[] copy = new RollbackOptimizedHistory[size];
for (int i = 0; i < size; i++) {
copy[i] = historyChanges.poll();
}
try (PreparedStatement stmt = connection.prepareStatement(INSERT_EDIT)) {
for (RollbackOptimizedHistory change : copy) {
// `player`,`id`,`x1`,`y1`,`z1`,`x2`,`y2`,`z2`,`time`
UUID uuid = change.getUUID();
byte[] uuidBytes = ByteBuffer.allocate(16).putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits()).array();
stmt.setBytes(1, uuidBytes);
stmt.setInt(2, change.getIndex());
stmt.setInt(3, change.getMinX());
stmt.setByte(4, (byte) (change.getMinY() - 128));
stmt.setInt(5, change.getMinZ());
stmt.setInt(6, change.getMaxX());
stmt.setByte(7, (byte) (change.getMaxY() - 128));
stmt.setInt(8, change.getMaxZ());
stmt.setInt(9, (int) (change.getTime() / 1000));
stmt.executeUpdate();
stmt.clearParameters();
}
} catch (Exception e) {
e.printStackTrace();
}
commit();
return true;
} catch (final Exception e) {
e.printStackTrace();
}
return false;
}
public void commit() {
try {
if (connection == null) {
return;
}
if (!connection.getAutoCommit()) {
connection.commit();
connection.setAutoCommit(true);
}
} catch (final SQLException e) {
e.printStackTrace();
}
}
public Connection openConnection() throws SQLException, ClassNotFoundException {
if (checkConnection()) {
return connection;
}
if (!Fawe.imp().getDirectory().exists()) {
Fawe.imp().getDirectory().mkdirs();
}
if (!(dbLocation.exists())) {
try {
dbLocation.getParentFile().mkdirs();
dbLocation.createNewFile();
} catch (final IOException e) {
e.printStackTrace();
Fawe.debug("&cUnable to create database!");
}
}
Class.forName("org.sqlite.JDBC");
connection = DriverManager.getConnection("jdbc:sqlite:" + dbLocation);
return connection;
}
public Connection forceConnection() throws SQLException, ClassNotFoundException {
Class.forName("org.sqlite.JDBC");
connection = DriverManager.getConnection("jdbc:sqlite:" + dbLocation);
return connection;
}
/**
* Gets the connection with the database
*
* @return Connection with the database, null if none
*/
public Connection getConnection() {
if (connection == null) {
try {
forceConnection();
} catch (final ClassNotFoundException e) {
e.printStackTrace();
} catch (final SQLException e) {
e.printStackTrace();
}
}
return connection;
}
/**
* Closes the connection with the database
*
* @return true if successful
* @throws java.sql.SQLException if the connection cannot be closed
*/
public boolean closeConnection() throws SQLException {
if (connection == null) {
return false;
}
connection.close();
connection = null;
return true;
}
/**
* Checks if a connection is open with the database
*
* @return true if the connection is open
* @throws java.sql.SQLException if the connection cannot be checked
*/
public boolean checkConnection() {
try {
return (connection != null) && !connection.isClosed();
} catch (final SQLException e) {
return false;
}
}
}

7
core/src/main/java/com/boydti/fawe/example/MappedFaweQueue.java
View File

@ -18,6 +18,8 @@ import com.sk89q.worldedit.world.biome.BaseBiome;
import com.sk89q.worldedit.world.registry.BundledBlockData;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
@ -41,6 +43,11 @@ public abstract class MappedFaweQueue<WORLD, CHUNK, SECTION> extends FaweQueue {
};
public ArrayDeque<Runnable> tasks = new ArrayDeque<>();
@Override
public Collection<FaweChunk> getFaweChunks() {
return Collections.unmodifiableCollection(chunks);
}
@Override
public void optimize() {
ArrayList<Thread> threads = new ArrayList<Thread>();

4
core/src/main/java/com/boydti/fawe/logging/rollback/RollbackDatabase.java Normal file
View File

@ -0,0 +1,4 @@
package com.boydti.fawe.logging.rollback;
public class RollbackDatabase {
}

95
core/src/main/java/com/boydti/fawe/logging/rollback/RollbackOptimizedHistory.java Normal file
View File

@ -0,0 +1,95 @@
package com.boydti.fawe.logging.rollback;
import com.boydti.fawe.object.changeset.DiskStorageHistory;
import com.sk89q.worldedit.world.World;
import java.io.IOException;
import java.util.UUID;
public class RollbackOptimizedHistory extends DiskStorageHistory {
private final long time;
private int minX;
private int maxX;
private int minY;
private int maxY;
private int minZ;
private int maxZ;
public RollbackOptimizedHistory(World world, UUID uuid, int index) {
super(world, uuid, index);
this.time = System.currentTimeMillis();
}
public RollbackOptimizedHistory(World world, UUID uuid) {
super(world, uuid);
this.time = System.currentTimeMillis();
}
public long getTime() {
return time;
}
public int getMinX() {
return minX;
}
public int getMaxX() {
return maxX;
}
public int getMinY() {
return minY;
}
public int getMaxY() {
return maxY;
}
public int getMinZ() {
return minZ;
}
public int getMaxZ() {
return maxZ;
}
@Override
public boolean flush() {
if (super.flush()) {
// Save to DB
return true;
}
return false;
}
@Override
public void add(int x, int y, int z, int combinedFrom, int combinedTo) {
super.add(x, y, z, combinedFrom, combinedTo);
if (x < minX) {
minX = x;
} else if (x > maxX) {
maxX = x;
}
if (y < minY) {
minY = y;
} else if (y > maxY) {
maxY = y;
}
if (z < minZ) {
minZ = z;
} else if (z > maxZ) {
maxZ = z;
}
}
@Override
public void writeHeader(int x, int y, int z) throws IOException {
minX = x;
maxX = x;
minY = y;
maxY = y;
minZ = z;
maxZ = z;
super.writeHeader(x, y, z);
}
}

3
core/src/main/java/com/boydti/fawe/object/FaweQueue.java
View File

@ -17,6 +17,7 @@ import com.sk89q.worldedit.blocks.BlockMaterial;
import com.sk89q.worldedit.world.biome.BaseBiome;
import com.sk89q.worldedit.world.registry.BundledBlockData;
import java.io.File;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
@ -133,6 +134,8 @@ public abstract class FaweQueue {
public abstract FaweChunk<?> getFaweChunk(int x, int z);
public abstract Collection<FaweChunk> getFaweChunks();
public abstract void setChunk(final FaweChunk<?> chunk);
public abstract File getSaveFolder();

16
core/src/main/java/com/boydti/fawe/object/change/MutablePlayerBlockChange.java Normal file
View File

@ -0,0 +1,16 @@
package com.boydti.fawe.object.change;
import java.util.UUID;
public class MutablePlayerBlockChange extends MutableBlockChange {
private final UUID uuid;
public MutablePlayerBlockChange(UUID uuid, int x, int y, int z, short id, byte data) {
super(x, y, z, id, data);
this.uuid = uuid;
}
public UUID getUIID() {
return uuid;
}
}

13
core/src/main/java/com/boydti/fawe/object/changeset/DiskStorageHistory.java
View File

@ -63,6 +63,8 @@ public class DiskStorageHistory extends FaweStreamChangeSet {
// Entity Create To
private NBTOutputStream osENTCT;
private int index;
public void deleteFiles() {
bdFile.delete();
nbtfFile.delete();
@ -97,6 +99,7 @@ public class DiskStorageHistory extends FaweStreamChangeSet {
private void init(UUID uuid, int i) {
this.uuid = uuid;
this.index = i;
String base = "history" + File.separator + Fawe.imp().getWorldName(getWorld()) + File.separator + uuid;
base += File.separator + i;
nbtfFile = new File(Fawe.imp().getDirectory(), base + ".nbtf");
@ -114,6 +117,10 @@ public class DiskStorageHistory extends FaweStreamChangeSet {
return bdFile;
}
public int getIndex() {
return index;
}
@Override
public boolean flush() {
super.flush();
@ -160,6 +167,11 @@ public class DiskStorageHistory extends FaweStreamChangeSet {
if (osBD != null) {
return osBD;
}
writeHeader(x, y, z);
return osBD;
}
public void writeHeader(int x, int y, int z) throws IOException {
bdFile.getParentFile().mkdirs();
bdFile.createNewFile();
osBD = getCompressedOS(new FileOutputStream(bdFile));
@ -175,7 +187,6 @@ public class DiskStorageHistory extends FaweStreamChangeSet {
osBD.write((byte) (z >> 16));
osBD.write((byte) (z >> 8));
osBD.write((byte) (z));
return osBD;
}
@Override

10
core/src/main/java/com/boydti/fawe/object/changeset/FaweChangeSet.java
View File

@ -2,6 +2,7 @@ package com.boydti.fawe.object.changeset;
import com.boydti.fawe.Fawe;
import com.boydti.fawe.FaweCache;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.object.BytePair;
import com.boydti.fawe.object.FaweChunk;
import com.boydti.fawe.object.FawePlayer;
@ -25,12 +26,21 @@ import com.sk89q.worldedit.world.World;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
public abstract class FaweChangeSet implements ChangeSet {
private final World world;
public static FaweChangeSet getDefaultChangeSet(World world, UUID uuid) {
if (Settings.HISTORY.USE_DISK) {
return new DiskStorageHistory(world, uuid);
} else {
return new MemoryOptimizedHistory(world);
}
}
public FaweChangeSet(World world) {
this.world = world;
}

6
core/src/main/java/com/boydti/fawe/util/EditSessionBuilder.java
View File

@ -1,6 +1,8 @@
package com.boydti.fawe.util;
import com.boydti.fawe.FaweAPI;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.logging.rollback.RollbackOptimizedHistory;
import com.boydti.fawe.object.FaweLimit;
import com.boydti.fawe.object.FawePlayer;
import com.boydti.fawe.object.NullChangeSet;
@ -89,7 +91,11 @@ public class EditSessionBuilder {
*/
public EditSessionBuilder changeSet(boolean disk, @Nullable UUID uuid, int compression) {
if (disk) {
if (Settings.HISTORY.USE_DATABASE) {
this.changeSet = new RollbackOptimizedHistory(world, uuid);
} else {
this.changeSet = new DiskStorageHistory(world, uuid);
}
} else {
this.changeSet = new MemoryOptimizedHistory(world);
}

2
core/src/main/java/com/boydti/fawe/util/TaskManager.java
View File

@ -123,7 +123,7 @@ public abstract class TaskManager {
public void taskNow(final Runnable r, boolean async) {
if (async) {
async(r);
} else {
} else if (r != null){
r.run();
}
}

5
core/src/main/java/com/sk89q/worldedit/EditSession.java
View File

@ -24,6 +24,7 @@ import com.boydti.fawe.FaweCache;
import com.boydti.fawe.config.BBC;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.logging.LoggingChangeSet;
import com.boydti.fawe.logging.rollback.RollbackOptimizedHistory;
import com.boydti.fawe.object.FaweLimit;
import com.boydti.fawe.object.FawePlayer;
import com.boydti.fawe.object.FaweQueue;
@ -196,7 +197,11 @@ public class EditSession implements Extent {
if (changeSet == null) {
if (Settings.HISTORY.USE_DISK) {
UUID uuid = player == null ? CONSOLE : player.getUUID();
if (Settings.HISTORY.USE_DATABASE) {
changeSet = new RollbackOptimizedHistory(world, uuid);
} else {
changeSet = new DiskStorageHistory(world, uuid);
}
} else if (Settings.HISTORY.COMBINE_STAGES && Settings.HISTORY.COMPRESSION_LEVEL == 0) {
changeSet = new CPUOptimizedChangeSet(world);
} else {

8
favs/src/main/java/com/thevoxelbox/voxelsniper/Sniper.java
View File

@ -4,16 +4,13 @@ import com.boydti.fawe.Fawe;
import com.boydti.fawe.FaweAPI;
import com.boydti.fawe.bukkit.wrapper.AsyncWorld;
import com.boydti.fawe.config.BBC;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.logging.LoggingChangeSet;
import com.boydti.fawe.object.ChangeSetFaweQueue;
import com.boydti.fawe.object.FawePlayer;
import com.boydti.fawe.object.FaweQueue;
import com.boydti.fawe.object.MaskedFaweQueue;
import com.boydti.fawe.object.RegionWrapper;
import com.boydti.fawe.object.changeset.DiskStorageHistory;
import com.boydti.fawe.object.changeset.FaweChangeSet;
import com.boydti.fawe.object.changeset.MemoryOptimizedHistory;
import com.boydti.fawe.util.StringMan;
import com.boydti.fawe.util.TaskManager;
import com.boydti.fawe.util.WEManager;
@ -95,7 +92,7 @@ public class Sniper {
RegionWrapper[] mask = WEManager.IMP.getMask(fp);
this.maskQueue = new MaskedFaweQueue(baseQueue, mask);
com.sk89q.worldedit.world.World worldEditWorld = fp.getWorld();
FaweChangeSet changeSet = Settings.HISTORY.USE_DISK ? new DiskStorageHistory(worldEditWorld, fp.getUUID()) : new MemoryOptimizedHistory(worldEditWorld);
FaweChangeSet changeSet = FaweChangeSet.getDefaultChangeSet(worldEditWorld, fp.getUUID());
if (Fawe.imp().getBlocksHubApi() != null) {
changeSet = LoggingChangeSet.wrap(fp, changeSet);
}
@ -407,7 +404,8 @@ public class Sniper {
session.remember(changeSet.toEditSession(fp));
changeQueue.flush();
com.sk89q.worldedit.world.World worldEditWorld = fp.getWorld();
changeQueue.setChangeSet(Settings.HISTORY.USE_DISK ? new DiskStorageHistory(worldEditWorld, fp.getUUID()) : new MemoryOptimizedHistory(worldEditWorld));
changeSet = FaweChangeSet.getDefaultChangeSet(worldEditWorld, fp.getUUID());
changeQueue.setChangeSet(changeSet);
}
}