From d0b5dab2a0a43db5aa1d0c684b4f275966ce6d03 Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Thu, 11 Aug 2016 21:34:56 +1000 Subject: [PATCH] Various Rename Favs jar and print error on failed load LocalSession history on disk size limit per player (in MB) Possible fix for ForgeEssentials incompatibility --- .../java/com/boydti/fawe/config/Settings.java | 9 +- .../fawe/database/RollbackDatabase.java | 17 ++++ .../com/boydti/fawe/object/FaweLimit.java | 2 +- .../object/changeset/DiskStorageHistory.java | 21 +++-- .../fawe/object/changeset/FaweChangeSet.java | 2 + .../java/com/boydti/fawe/util/MainUtil.java | 48 ++++++++++- .../com/sk89q/worldedit/LocalSession.java | 84 +++++++++++++++++-- favs/build.gradle | 2 +- .../com/boydti/fawe/bukkit/favs/Favs.java | 4 +- 9 files changed, 168 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/boydti/fawe/config/Settings.java b/core/src/main/java/com/boydti/fawe/config/Settings.java index a1d226aa..fdf6afc0 100644 --- a/core/src/main/java/com/boydti/fawe/config/Settings.java +++ b/core/src/main/java/com/boydti/fawe/config/Settings.java @@ -71,8 +71,11 @@ public class Settings extends Config { "NoteBlock, Sign, Skull, Structure" }) public int MAX_BLOCKSTATES = 1337; - @Comment("Maximum size of the player's history in Megabytes") - public int MAX_HISTORY_MB = 200; + @Comment({ + "Maximum size of the player's history in Megabytes:", + " - History on disk or memory will be deleted", + }) + public int MAX_HISTORY_MB = -1; } public static class HISTORY { @@ -266,7 +269,7 @@ public class Settings extends Config { limit.MAX_ENTITIES = Math.max(limit.MAX_ENTITIES, newLimit.MAX_ENTITIES != -1 ? newLimit.MAX_ENTITIES : Integer.MAX_VALUE); limit.MAX_FAILS = Math.max(limit.MAX_FAILS, newLimit.MAX_FAILS != -1 ? newLimit.MAX_FAILS : Integer.MAX_VALUE); limit.MAX_ITERATIONS = Math.max(limit.MAX_ITERATIONS, newLimit.MAX_ITERATIONS != -1 ? newLimit.MAX_ITERATIONS : Integer.MAX_VALUE); - limit.MAX_HISTORY = HISTORY.USE_DISK ? Integer.MAX_VALUE : Math.max(limit.MAX_HISTORY, newLimit.MAX_HISTORY_MB != -1 ? newLimit.MAX_HISTORY_MB : Integer.MAX_VALUE); + limit.MAX_HISTORY = Math.max(limit.MAX_HISTORY, newLimit.MAX_HISTORY_MB != -1 ? newLimit.MAX_HISTORY_MB : Integer.MAX_VALUE); } } return limit; diff --git a/core/src/main/java/com/boydti/fawe/database/RollbackDatabase.java b/core/src/main/java/com/boydti/fawe/database/RollbackDatabase.java index b6e221a5..ae9b33dd 100644 --- a/core/src/main/java/com/boydti/fawe/database/RollbackDatabase.java +++ b/core/src/main/java/com/boydti/fawe/database/RollbackDatabase.java @@ -35,6 +35,7 @@ public class RollbackDatabase { private String GET_EDITS; private String GET_EDITS_USER; private String DELETE_EDITS_USER; + private String DELETE_EDIT_USER; private String PURGE; private ConcurrentLinkedQueue historyChanges = new ConcurrentLinkedQueue<>(); @@ -52,6 +53,7 @@ public class RollbackDatabase { GET_EDITS = "SELECT `player`,`id` FROM `" + prefix + "edits` WHERE `x2`>=? AND `x1`<=? AND `y2`>=? AND `y1`<=? AND `z2`>=? AND `z1`<=? AND `time`>? ORDER BY `time` DESC, `id` DESC"; GET_EDITS_USER = "SELECT `player`,`id` FROM `" + prefix + "edits` WHERE `x2`>=? AND `x1`<=? AND `y2`>=? AND `y1`<=? AND `z2`>=? AND `z1`<=? AND `time`>? AND `player`=? ORDER BY `time` DESC, `id` DESC"; DELETE_EDITS_USER = "DELETE FROM `" + prefix + "edits` WHERE `x2`>=? AND `x1`<=? AND `y2`>=? AND `y1`<=? AND `z2`>=? AND `z1`<=? AND `time`>? AND `player`=?"; + DELETE_EDIT_USER = "DELETE FROM `" + prefix + "edits` WHERE `player`=? AND `id`=?"; init(); purge((int) TimeUnit.DAYS.toMillis(Settings.HISTORY.DELETE_AFTER_DAYS)); TaskManager.IMP.async(new Runnable() { @@ -90,6 +92,21 @@ public class RollbackDatabase { notify.add(run); } + public void delete(final UUID uuid, final int id) { + addTask(new Runnable() { + @Override + public void run() { + try (PreparedStatement stmt = connection.prepareStatement(DELETE_EDIT_USER)) { + byte[] uuidBytes = ByteBuffer.allocate(16).putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits()).array(); + stmt.setBytes(1, uuidBytes); + stmt.setInt(2, id); + } catch (SQLException e) { + e.printStackTrace(); + } + } + }); + } + public void purge(int diff) { long now = System.currentTimeMillis() / 1000; final int then = (int) (now - diff); diff --git a/core/src/main/java/com/boydti/fawe/object/FaweLimit.java b/core/src/main/java/com/boydti/fawe/object/FaweLimit.java index 8cf8bc49..9b3bbbe4 100644 --- a/core/src/main/java/com/boydti/fawe/object/FaweLimit.java +++ b/core/src/main/java/com/boydti/fawe/object/FaweLimit.java @@ -47,7 +47,7 @@ public class FaweLimit { MAX.MAX_ITERATIONS = Integer.MAX_VALUE; MAX.MAX_BLOCKSTATES = Integer.MAX_VALUE; MAX.MAX_ENTITIES = Integer.MAX_VALUE; - MAX.MAX_HISTORY = 15; + MAX.MAX_HISTORY = Integer.MAX_VALUE; } public boolean MAX_CHANGES() { diff --git a/core/src/main/java/com/boydti/fawe/object/changeset/DiskStorageHistory.java b/core/src/main/java/com/boydti/fawe/object/changeset/DiskStorageHistory.java index 4ff8d9cb..d4649aa6 100644 --- a/core/src/main/java/com/boydti/fawe/object/changeset/DiskStorageHistory.java +++ b/core/src/main/java/com/boydti/fawe/object/changeset/DiskStorageHistory.java @@ -2,6 +2,8 @@ package com.boydti.fawe.object.changeset; import com.boydti.fawe.Fawe; import com.boydti.fawe.config.Settings; +import com.boydti.fawe.database.DBHandler; +import com.boydti.fawe.database.RollbackDatabase; import com.boydti.fawe.object.FaweInputStream; import com.boydti.fawe.object.IntegerPair; import com.boydti.fawe.object.RegionWrapper; @@ -101,6 +103,15 @@ public class DiskStorageHistory extends FaweStreamChangeSet { initFiles(folder); } + public void delete() { + Fawe.debug("Deleting history: " + Fawe.imp().getWorldName(getWorld()) + "/" + uuid + "/" + index); + deleteFiles(); + if (Settings.HISTORY.USE_DATABASE) { + RollbackDatabase db = DBHandler.IMP.getDatabase(Fawe.imp().getWorldName(getWorld())); + db.delete(uuid, index); + } + } + public void deleteFiles() { bdFile.delete(); nbtfFile.delete(); @@ -166,19 +177,19 @@ public class DiskStorageHistory extends FaweStreamChangeSet { public long getSizeOnDisk() { int total = 0; if (bdFile.exists()) { - total += bdFile.getTotalSpace(); + total += bdFile.length(); } if (nbtfFile.exists()) { - total += entfFile.getTotalSpace(); + total += entfFile.length(); } if (nbttFile.exists()) { - total += entfFile.getTotalSpace(); + total += entfFile.length(); } if (entfFile.exists()) { - total += entfFile.getTotalSpace(); + total += entfFile.length(); } if (enttFile.exists()) { - total += entfFile.getTotalSpace(); + total += entfFile.length(); } return total; } diff --git a/core/src/main/java/com/boydti/fawe/object/changeset/FaweChangeSet.java b/core/src/main/java/com/boydti/fawe/object/changeset/FaweChangeSet.java index a91d1510..19a2960d 100644 --- a/core/src/main/java/com/boydti/fawe/object/changeset/FaweChangeSet.java +++ b/core/src/main/java/com/boydti/fawe/object/changeset/FaweChangeSet.java @@ -80,6 +80,8 @@ public abstract class FaweChangeSet implements ChangeSet { public abstract void addEntityCreate(CompoundTag tag); public abstract Iterator getIterator(boolean redo); + public void delete() {}; + public EditSession toEditSession(FawePlayer player) { EditSession editSession = new EditSessionBuilder(world).player(player).autoQueue(false).fastmode(false).checkMemory(false).changeSet(this).allowedRegions(RegionWrapper.GLOBAL().toArray()).build(); editSession.setSize(1); diff --git a/core/src/main/java/com/boydti/fawe/util/MainUtil.java b/core/src/main/java/com/boydti/fawe/util/MainUtil.java index 5ff25710..2b8b8483 100644 --- a/core/src/main/java/com/boydti/fawe/util/MainUtil.java +++ b/core/src/main/java/com/boydti/fawe/util/MainUtil.java @@ -8,6 +8,7 @@ import com.boydti.fawe.object.FaweOutputStream; import com.boydti.fawe.object.FawePlayer; import com.boydti.fawe.object.RegionWrapper; import com.boydti.fawe.object.RunnableVal; +import com.boydti.fawe.object.RunnableVal2; import com.boydti.fawe.object.changeset.CPUOptimizedChangeSet; import com.boydti.fawe.object.changeset.FaweStreamChangeSet; import com.sk89q.jnbt.CompoundTag; @@ -38,12 +39,18 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; @@ -78,6 +85,42 @@ public class MainUtil { Fawe.debug(s); } + public static long getTotalSize(Path path) { + final AtomicLong size = new AtomicLong(0); + traverse(path, new RunnableVal2() { + @Override + public void run(Path path, BasicFileAttributes attrs) { + size.addAndGet (attrs.size()); + } + }); + return size.get(); + } + + public static long traverse(Path path, final RunnableVal2 onEach) { + final AtomicLong size = new AtomicLong(0); + try { + Files.walkFileTree(path, new SimpleFileVisitor() { + @Override public FileVisitResult + visitFile(Path file, BasicFileAttributes attrs) { + onEach.run(file, attrs); + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult + visitFileFailed(Path file, IOException exc) { + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult + postVisitDirectory (Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + }); + } + catch (IOException e) { + throw new AssertionError ("walkFileTree will not throw IOException if the FileVisitor does not"); + } + return size.get(); + } + public static File getFile(File base, String path) { if (Paths.get(path).isAbsolute()) { return new File(path); @@ -93,9 +136,10 @@ public class MainUtil { return getCompressedOS(os, Settings.HISTORY.COMPRESSION_LEVEL); } - public static long getSizeInMemory(ChangeSet changeSet) { + public static long getSize(ChangeSet changeSet) { if (changeSet instanceof FaweStreamChangeSet){ - return 92 + ((FaweStreamChangeSet) changeSet).getSizeInMemory(); + FaweStreamChangeSet fscs = (FaweStreamChangeSet) changeSet; + return fscs.getSizeOnDisk() + fscs.getSizeInMemory(); } else if (changeSet instanceof CPUOptimizedChangeSet) { return changeSet.size() + 32; } else if (changeSet != null) { diff --git a/core/src/main/java/com/sk89q/worldedit/LocalSession.java b/core/src/main/java/com/sk89q/worldedit/LocalSession.java index 0f32e57b..d1383650 100644 --- a/core/src/main/java/com/sk89q/worldedit/LocalSession.java +++ b/core/src/main/java/com/sk89q/worldedit/LocalSession.java @@ -24,11 +24,13 @@ import com.boydti.fawe.config.Settings; import com.boydti.fawe.object.FaweInputStream; import com.boydti.fawe.object.FaweOutputStream; import com.boydti.fawe.object.FawePlayer; +import com.boydti.fawe.object.RunnableVal2; import com.boydti.fawe.object.changeset.DiskStorageHistory; import com.boydti.fawe.object.changeset.FaweChangeSet; import com.boydti.fawe.object.clipboard.DiskOptimizedClipboard; import com.boydti.fawe.util.EditSessionBuilder; import com.boydti.fawe.util.MainUtil; +import com.boydti.fawe.wrappers.WorldWrapper; import com.sk89q.jchronic.Chronic; import com.sk89q.jchronic.Options; import com.sk89q.jchronic.utils.Span; @@ -61,6 +63,10 @@ import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; @@ -72,6 +78,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; import javax.swing.filechooser.FileNameExtensionFilter; @@ -118,7 +125,6 @@ public class LocalSession { } }); private transient volatile Integer historyNegativeIndex; - private transient volatile long historySize = 0; private transient ClipboardHolder clipboard; private transient boolean toolControl = true; private transient boolean superPickaxe = false; @@ -133,9 +139,9 @@ public class LocalSession { private transient Mask mask; private transient TimeZone timezone = TimeZone.getDefault(); - // May be null private transient World currentWorld; private transient UUID uuid; + private transient volatile long historySize = 0; // Saved properties private String lastScript; @@ -222,16 +228,70 @@ public class LocalSession { } }); } - historySize = 0; if (editIds.size() > 0) { + historySize = MainUtil.getTotalSize(folder.toPath()); Collections.sort(editIds); for (int index : editIds) { history.add(index); } + } else { + historySize = 0; } return editIds.size() > 0; } + @Deprecated + private void deleteOldFiles(UUID uuid, World world, long maxBytes) throws IOException { + final File folder = MainUtil.getFile(Fawe.imp().getDirectory(), Settings.PATHS.HISTORY + File.separator + Fawe.imp().getWorldName(world) + File.separator + uuid); + + final ArrayList ids = new ArrayList(); + final HashMap> paths = new HashMap<>(); + final HashMap sizes = new HashMap<>(); + final AtomicLong totalSize = new AtomicLong(); + + MainUtil.traverse(folder.toPath(), new RunnableVal2() { + @Override + public void run(Path path, BasicFileAttributes attr) { + try { + String file = path.getFileName().toString(); + int index = file.indexOf('.'); + if (index == -1) { + return; + } + int id = Integer.parseInt(file.substring(0, index)); + long size = attr.size(); + totalSize.addAndGet(size); + ArrayDeque existingPaths = paths.get(id); + if (existingPaths == null) { + existingPaths = new ArrayDeque(); + paths.put(id, existingPaths); + } + existingPaths.add(path); + AtomicLong existingSize = sizes.get(id); + if (existingSize == null) { + existingSize = new AtomicLong(); + sizes.put(id, existingSize); + } + existingSize.addAndGet(size); + } catch (NumberFormatException ignore){ + ignore.printStackTrace(); + } + } + }); + if (totalSize.get() < maxBytes) { + return; + } + Collections.sort(ids); + long total = totalSize.get(); + for (int i = 0; i < ids.size() && total > maxBytes; i++) { + int id = ids.get(i); + for (Path path : paths.get(id)) { + Files.delete(path); + } + total -= sizes.get(id).get(); + } + } + private void loadHistoryNegativeIndex(UUID uuid, World world) { File file = MainUtil.getFile(Fawe.imp().getDirectory(), Settings.PATHS.HISTORY + File.separator + Fawe.imp().getWorldName(world) + File.separator + uuid + File.separator + "index"); if (file.exists()) { @@ -392,16 +452,19 @@ public class LocalSession { while (iter.hasNext()) { Object item = iter.next(); if (++i > cutoffIndex) { + FaweChangeSet changeSet; if (item instanceof FaweChangeSet) { - FaweChangeSet changeSet = (FaweChangeSet) item; - historySize -= MainUtil.getSizeInMemory(changeSet); + changeSet = (FaweChangeSet) item; + } else { + changeSet = getChangeSet(item); } + historySize -= MainUtil.getSize(changeSet); iter.remove(); } } } FaweChangeSet changeSet = (FaweChangeSet) editSession.getChangeSet(); - historySize += MainUtil.getSizeInMemory(changeSet); + historySize += MainUtil.getSize(changeSet); if (append) { history.add(changeSet); if (getHistoryNegativeIndex() != 0) { @@ -413,7 +476,8 @@ public class LocalSession { } while ((history.size() > MAX_HISTORY_SIZE || (historySize >> 20) > limitMb) && history.size() > 1) { FaweChangeSet item = (FaweChangeSet) history.remove(0); - historySize -= MainUtil.getSizeInMemory(item); + item.delete(); + historySize -= MainUtil.getSize(item); } } @@ -648,7 +712,11 @@ public class LocalSession { * @return the the world of the selection */ public World getSelectionWorld() { - return selector.getIncompleteRegion().getWorld(); + World world = selector.getIncompleteRegion().getWorld(); + if (world instanceof WorldWrapper) { + return ((WorldWrapper) world).getParent(); + } + return world; } /** diff --git a/favs/build.gradle b/favs/build.gradle index d563da34..5f6823b8 100644 --- a/favs/build.gradle +++ b/favs/build.gradle @@ -14,4 +14,4 @@ processResources { } jar.destinationDir = file '../target' -jar.archiveName = "${parent.name}-${project.name}-${parent.version}.jar" \ No newline at end of file +jar.archiveName = "FastAsyncVoxelSniper-${project.name}-${parent.version}.jar" \ No newline at end of file diff --git a/favs/src/main/java/com/boydti/fawe/bukkit/favs/Favs.java b/favs/src/main/java/com/boydti/fawe/bukkit/favs/Favs.java index 219a1c4e..40752a5c 100644 --- a/favs/src/main/java/com/boydti/fawe/bukkit/favs/Favs.java +++ b/favs/src/main/java/com/boydti/fawe/bukkit/favs/Favs.java @@ -45,6 +45,8 @@ public class Favs extends JavaPlugin { } }); Fawe.debug("Injected VoxelSniper classes"); - } catch (Throwable ignore) {} + } catch (Throwable ignore) { + ignore.printStackTrace(); + } } }