diff --git a/bukkit110/src/main/java/com/boydti/fawe/bukkit/v1_10/BukkitQueue_1_10.java b/bukkit110/src/main/java/com/boydti/fawe/bukkit/v1_10/BukkitQueue_1_10.java index fedcbbb7..bd2ae6d8 100644 --- a/bukkit110/src/main/java/com/boydti/fawe/bukkit/v1_10/BukkitQueue_1_10.java +++ b/bukkit110/src/main/java/com/boydti/fawe/bukkit/v1_10/BukkitQueue_1_10.java @@ -364,8 +364,12 @@ public class BukkitQueue_1_10 extends BukkitQueue_0 extends FaweQueue { public MappedFaweQueue(final String world) { super(world); - map = new DefaultFaweQueueMap(this); + map = Settings.PREVENT_CRASHES ? new WeakFaweQueueMap(this) : new DefaultFaweQueueMap(this); + } + + public MappedFaweQueue(final String world, IFaweQueueMap map) { + super(world); + if (map == null) { + map = Settings.PREVENT_CRASHES ? new WeakFaweQueueMap(this) : new DefaultFaweQueueMap(this); + } + this.map = map; } public MappedFaweQueue(final World world, IFaweQueueMap map) { super(world); if (map == null) { - map = new DefaultFaweQueueMap(this); + map = Settings.PREVENT_CRASHES ? new WeakFaweQueueMap(this) : new DefaultFaweQueueMap(this); } this.map = map; } diff --git a/core/src/main/java/com/boydti/fawe/example/NMSMappedFaweQueue.java b/core/src/main/java/com/boydti/fawe/example/NMSMappedFaweQueue.java index 9be3ee08..6d0e267a 100644 --- a/core/src/main/java/com/boydti/fawe/example/NMSMappedFaweQueue.java +++ b/core/src/main/java/com/boydti/fawe/example/NMSMappedFaweQueue.java @@ -30,6 +30,12 @@ public abstract class NMSMappedFaweQueue ex addRelightTask(); } + public NMSMappedFaweQueue(String world, IFaweQueueMap map) { + super(world, map); + this.maxY = 256; + addRelightTask(); + } + public NMSMappedFaweQueue(World world, IFaweQueueMap map) { super(world, map); this.maxY = world.getMaxY(); diff --git a/core/src/main/java/com/boydti/fawe/example/WeakFaweQueueMap.java b/core/src/main/java/com/boydti/fawe/example/WeakFaweQueueMap.java new file mode 100644 index 00000000..57f8b7cd --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/example/WeakFaweQueueMap.java @@ -0,0 +1,214 @@ +package com.boydti.fawe.example; + +import com.boydti.fawe.Fawe; +import com.boydti.fawe.object.FaweChunk; +import com.boydti.fawe.object.FaweQueue; +import com.boydti.fawe.object.RunnableVal; +import com.boydti.fawe.util.MathMan; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorCompletionService; + +public class WeakFaweQueueMap implements IFaweQueueMap { + + private final MappedFaweQueue parent; + + public WeakFaweQueueMap(MappedFaweQueue parent) { + this.parent = parent; + } + + /** + * Map of chunks in the queue + */ + public ConcurrentHashMap> blocks = new ConcurrentHashMap>(8, 0.9f, 1) { + @Override + public Reference put(Long key, Reference value) { + if (parent.getProgressTask() != null) { + try { + parent.getProgressTask().run(FaweQueue.ProgressType.QUEUE, size() + 1); + } catch (Throwable e) { + e.printStackTrace(); + } + } + return super.put(key, value); + } + }; + + @Override + public Collection getFaweCunks() { + HashSet set = new HashSet<>(); + Iterator>> iter = blocks.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + FaweChunk value = entry.getValue().get(); + if (value != null) { + set.add(value); + } else { + Fawe.debug("Skipped modifying chunk due to low memory (1)"); + iter.remove(); + } + } + return set; + } + + @Override + public void forEachChunk(RunnableVal onEach) { + Iterator>> iter = blocks.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + FaweChunk value = entry.getValue().get(); + if (value != null) { + onEach.run(value); + } else { + Fawe.debug("Skipped modifying chunk due to low memory (2)"); + iter.remove(); + } + } + } + + @Override + public FaweChunk getFaweChunk(int cx, int cz) { + if (cx == lastX && cz == lastZ) { + return lastWrappedChunk; + } + long pair = MathMan.pairInt(cx, cz); + Reference chunkReference = this.blocks.get(pair); + FaweChunk chunk; + if (chunkReference == null || (chunk = chunkReference.get()) == null) { + chunk = this.getNewFaweChunk(cx, cz); + Reference previous = this.blocks.put(pair, new SoftReference(chunk)); + if (previous != null) { + FaweChunk tmp = previous.get(); + if (tmp != null) { + chunk = tmp; + this.blocks.put(pair, previous); + } + } + + } + return chunk; + } + + @Override + public FaweChunk getCachedFaweChunk(int cx, int cz) { + if (cx == lastX && cz == lastZ) { + return lastWrappedChunk; + } + long pair = MathMan.pairInt(cx, cz); + Reference reference = this.blocks.get(pair); + if (reference != null) { + return reference.get(); + } else { + return null; + } + } + + @Override + public void add(FaweChunk chunk) { + long pair = MathMan.pairInt(chunk.getX(), chunk.getZ()); + Reference previous = this.blocks.put(pair, new SoftReference(chunk)); + if (previous != null) { + FaweChunk previousChunk = previous.get(); + if (previousChunk != null) { + blocks.put(pair, previous); + } + } + } + + + @Override + public void clear() { + blocks.clear(); + } + + @Override + public int size() { + return blocks.size(); + } + + private FaweChunk getNewFaweChunk(int cx, int cz) { + return parent.getFaweChunk(cx, cz); + } + + private FaweChunk lastWrappedChunk; + private int lastX = Integer.MIN_VALUE; + private int lastZ = Integer.MIN_VALUE; + + @Override + public boolean next(int amount, ExecutorCompletionService pool, long time) { + lastWrappedChunk = null; + lastX = Integer.MIN_VALUE; + lastZ = Integer.MIN_VALUE; + try { + int added = 0; + Iterator>> iter = blocks.entrySet().iterator(); + if (amount == 1) { + long start = System.currentTimeMillis(); + do { + if (iter.hasNext()) { + Map.Entry> entry = iter.next(); + Reference chunkReference = entry.getValue(); + FaweChunk chunk = chunkReference.get(); + iter.remove(); + if (chunk != null) { + parent.start(chunk); + chunk.call(); + parent.end(chunk); + } else { + Fawe.debug("Skipped modifying chunk due to low memory (3)"); + } + } else { + break; + } + } while (System.currentTimeMillis() - start < time); + return !blocks.isEmpty(); + } + boolean result = true; + // amount = 8; + for (int i = 0; i < amount && (result = iter.hasNext()); i++, added++) { + Map.Entry> item = iter.next(); + Reference chunkReference = item.getValue(); + FaweChunk chunk = chunkReference.get(); + iter.remove(); + if (chunk != null) { + parent.start(chunk); + pool.submit(chunk); + } else { + Fawe.debug("Skipped modifying chunk due to low memory (4)"); + i--; + added--; + } + } + // if result, then submitted = amount + if (result) { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < time && result) { + if (result = iter.hasNext()) { + Map.Entry> item = iter.next(); + Reference chunkReference = item.getValue(); + FaweChunk chunk = chunkReference.get(); + iter.remove(); + if (chunk != null) { + parent.start(chunk); + pool.submit(chunk); + FaweChunk fc = ((FaweChunk) pool.take().get()); + parent.end(fc); + } + } + } + } + for (int i = 0; i < added; i++) { + FaweChunk fc = ((FaweChunk) pool.take().get()); + parent.end(fc); + } + } catch (Throwable e) { + e.printStackTrace(); + } + return !blocks.isEmpty(); + } +} diff --git a/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java b/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java index d1906e87..69b83378 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java @@ -21,6 +21,7 @@ public class NBTStreamer { this.value2 = readers.get(node); } }); + is.close(); } public void addReader(String node, RunnableVal2 run) { diff --git a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java index c3ec6405..7abf6275 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java @@ -196,6 +196,10 @@ public class MCAChunk extends FaweChunk { this.deleted = deleted; } + public boolean isDeleted() { + return deleted; + } + public boolean isModified() { return modified; } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java index f71c760a..370fa96e 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java @@ -1,6 +1,5 @@ package com.boydti.fawe.jnbt.anvil; -import com.boydti.fawe.config.Settings; import com.boydti.fawe.jnbt.NBTStreamer; import com.boydti.fawe.object.FaweQueue; import com.boydti.fawe.object.RunnableVal; @@ -34,10 +33,10 @@ import java.util.zip.InflaterInputStream; */ public class MCAFile { - private final File file; - private final RandomAccessFile raf; - private final byte[] locations; - private final FaweQueue queue; + private File file; + private RandomAccessFile raf; + private byte[] locations; + private FaweQueue queue; private Field fieldBuf1; private Field fieldBuf2; private Field fieldBuf3; @@ -45,15 +44,15 @@ public class MCAFile { private Field fieldBuf5; private Field fieldBuf6; - private byte[] buffer1 = new byte[Settings.HISTORY.BUFFER_SIZE]; - private byte[] buffer2 = new byte[Settings.HISTORY.BUFFER_SIZE]; + private byte[] buffer1 = new byte[4096]; + private byte[] buffer2 = new byte[4096]; private byte[] buffer3 = new byte[720]; private final int X, Z; private Map chunks = new HashMap<>(); - public MCAFile(FaweQueue parent, File file) throws Exception { + public MCAFile(FaweQueue parent, File file) { this.queue = parent; this.file = file; if (!file.exists()) { @@ -62,21 +61,34 @@ public class MCAFile { String[] split = file.getName().split("\\."); X = Integer.parseInt(split[1]); Z = Integer.parseInt(split[2]); - this.locations = new byte[4096]; - this.raf = new BufferedRandomAccessFile(file, "rw", Settings.HISTORY.BUFFER_SIZE); - raf.readFully(locations); - fieldBuf1 = BufferedInputStream.class.getDeclaredField("buf"); - fieldBuf1.setAccessible(true); - fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf"); - fieldBuf2.setAccessible(true); - fieldBuf3 = NBTInputStream.class.getDeclaredField("buf"); - fieldBuf3.setAccessible(true); - fieldBuf4 = FastByteArrayOutputStream.class.getDeclaredField("array"); - fieldBuf4.setAccessible(true); - fieldBuf5 = DeflaterOutputStream.class.getDeclaredField("buf"); - fieldBuf5.setAccessible(true); - fieldBuf6 = BufferedOutputStream.class.getDeclaredField("buf"); - fieldBuf6.setAccessible(true); + } + + public FaweQueue getParent() { + return queue; + } + + public void init() { + try { + if (raf == null) { + this.locations = new byte[4096]; + this.raf = new BufferedRandomAccessFile(file, "rw", (int) file.length()); + raf.readFully(locations); + fieldBuf1 = BufferedInputStream.class.getDeclaredField("buf"); + fieldBuf1.setAccessible(true); + fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf"); + fieldBuf2.setAccessible(true); + fieldBuf3 = NBTInputStream.class.getDeclaredField("buf"); + fieldBuf3.setAccessible(true); + fieldBuf4 = FastByteArrayOutputStream.class.getDeclaredField("buffer"); + fieldBuf4.setAccessible(true); + fieldBuf5 = DeflaterOutputStream.class.getDeclaredField("buf"); + fieldBuf5.setAccessible(true); + fieldBuf6 = BufferedOutputStream.class.getDeclaredField("buf"); + fieldBuf6.setAccessible(true); + } + } catch (Throwable e) { + e.printStackTrace(); + } } public MCAFile(FaweQueue parent, int mcrX, int mcrZ) throws Exception { @@ -240,13 +252,6 @@ public class MCAFile { return new ArrayList<>(chunks.values()); } - private NBTStreamer getChunkReader(int offset) throws Exception { - if (offset == 0) { - return null; - } - return new NBTStreamer(getChunkIS(offset)); - } - public void uncache(int cx, int cz) { int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); chunks.remove(pair); @@ -254,12 +259,11 @@ public class MCAFile { private byte[] toBytes(MCAChunk chunk) throws Exception { CompoundTag tag = chunk.toTag(); - if (tag == null) { + if (tag == null || chunk.isDeleted()) { return null; } - FastByteArrayOutputStream baos = new FastByteArrayOutputStream(0); - fieldBuf4.set(baos, buffer3); - DeflaterOutputStream deflater = new DeflaterOutputStream(baos, new Deflater(6), 1, true); + FastByteArrayOutputStream baos = new FastByteArrayOutputStream(buffer3); + DeflaterOutputStream deflater = new DeflaterOutputStream(baos, new Deflater(9), 1, true); fieldBuf5.set(deflater, buffer2); BufferedOutputStream bos = new BufferedOutputStream(deflater, 1); fieldBuf6.set(bos, buffer1); @@ -268,6 +272,10 @@ public class MCAFile { bos.flush(); bos.close(); byte[] result = baos.toByteArray(); + baos.close(); + deflater.close(); + bos.close(); + nos.close(); return result; } @@ -290,14 +298,33 @@ public class MCAFile { raf.write((offsetMedium >> 8)); raf.write((offsetMedium >> 0)); raf.write(sizeByte); + + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i+ 2] & 0xFF))) << 12; + int size = (locations[i + 3] & 0xFF) << 12; } public void close() { flush(); - try { - raf.close(); - } catch (IOException e) { - e.printStackTrace(); + if (raf != null) { + try { + raf.close(); + } catch (IOException e) { + e.printStackTrace(); + } + file = null; + raf = null; + locations = null; + queue = null; + fieldBuf1 = null; + fieldBuf2 = null; + fieldBuf3 = null; + fieldBuf4 = null; + fieldBuf5 = null; + fieldBuf6 = null; + buffer1 = null; + buffer2 = null; + buffer3 = null; + chunks = null; } } @@ -324,6 +351,7 @@ public class MCAFile { HashMap relocate = new HashMap(); int start = 8192; + int written = start; int end = 8192; int nextOffset = 8192; try { @@ -347,6 +375,7 @@ public class MCAFile { MCAChunk cached = getCachedChunk(cx, cz); if (cached == null || !cached.isModified()) { start += size; + written = start; continue; } else { newBytes = toBytes(cached); @@ -356,7 +385,7 @@ public class MCAFile { } } if (newBytes == null) { - // Don't write + writeHeader(cx, cz, 0, 0); continue; } int len = newBytes.length + 5; @@ -383,11 +412,14 @@ public class MCAFile { // System.out.println("Header: " + cx + "," + cz + " | " + offset + "," + start + " | " + end + "," + (start + size) + " | " + size + " | " + start); writeHeader(cx, cz, start >> 12, newSize); } + written = start + newBytes.length + 6; start += newSize << 12; } + raf.setLength(written); if (raf instanceof BufferedRandomAccessFile) { ((BufferedRandomAccessFile) raf).flush(); } + raf.close(); } catch (Throwable e) { e.printStackTrace(); } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAQueue.java b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAQueue.java index 05b3ec1f..7961c063 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAQueue.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAQueue.java @@ -1,18 +1,17 @@ package com.boydti.fawe.jnbt.anvil; -import com.boydti.fawe.Fawe; import com.boydti.fawe.example.CharFaweChunk; import com.boydti.fawe.example.NMSMappedFaweQueue; import com.boydti.fawe.object.FaweChunk; import com.boydti.fawe.object.FaweQueue; import com.boydti.fawe.object.RunnableVal4; -import com.boydti.fawe.util.TaskManager; import com.sk89q.jnbt.CompoundTag; import java.io.File; import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; public class MCAQueue extends NMSMappedFaweQueue { @@ -34,7 +33,7 @@ public class MCAQueue extends NMSMappedFaweQueue extends AbstractMap - implements Serializable { - /** The internal HashMap that will hold the SoftReference. */ - private final Map> hash = - new HashMap>(); +/** + * A SoftHashMap is a memory-constrained map that stores its values in + * {@link SoftReference SoftReference}s. (Contrast this with the JDK's + * {@link WeakHashMap WeakHashMap}, which uses weak references for its keys, which is of little value if you + * want the cache to auto-resize itself based on memory constraints). + *

+ * Having the values wrapped by soft references allows the cache to automatically reduce its size based on memory + * limitations and garbage collection. This ensures that the cache will not cause memory leaks by holding strong + * references to all of its values. + *

+ * This class is a generics-enabled Map based on initial ideas from Heinz Kabutz's and Sydney Redelinghuys's + * publicly posted version (with their approval), with + * continued modifications. It was copied from the Apache Shiro framework. + *

+ * This implementation is thread-safe and usable in concurrent environments. + * + * @since 0.8 + * @see SoftReference + * @see Apache Shiro + */ +public class SoftHashMap implements Map { - private final Map, K> reverseLookup = - new HashMap, K>(); + /** + * The default value of the RETENTION_SIZE attribute, equal to 100. + */ + private static final int DEFAULT_RETENTION_SIZE = 100; - /** Reference queue for cleared SoftReference objects. */ - private final ReferenceQueue queue = new ReferenceQueue(); + /** + * The internal HashMap that will hold the SoftReference. + */ + private final Map> map; - public V get(Object key) { - expungeStaleEntries(); - V result = null; - // We get the SoftReference represented by that key - SoftReference soft_ref = hash.get(key); - if (soft_ref != null) { - // From the SoftReference we get the value, which can be - // null if it has been garbage collected - result = soft_ref.get(); - if (result == null) { - // If the value has been garbage collected, remove the - // entry from the HashMap. - hash.remove(key); - reverseLookup.remove(soft_ref); - } - } - return result; - } + /** + * The number of strong references to hold internally, that is, the number of instances to prevent + * from being garbage collected automatically (unlike other soft references). + */ + private final int RETENTION_SIZE; - private void expungeStaleEntries() { - Reference sv; - while ((sv = queue.poll()) != null) { - hash.remove(reverseLookup.remove(sv)); - } - } + /** + * The FIFO list of strong references (not to be garbage collected), order of last access. + */ + private final Queue strongReferences; //guarded by 'strongReferencesLock' + private final ReentrantLock strongReferencesLock; - public V put(K key, V value) { - expungeStaleEntries(); - SoftReference soft_ref = new SoftReference(value, queue); - reverseLookup.put(soft_ref, key); - SoftReference result = hash.put(key, soft_ref); - if (result == null) return null; - reverseLookup.remove(result); - return result.get(); - } + /** + * Reference queue for cleared SoftReference objects. + */ + private final ReferenceQueue queue; - public V remove(Object key) { - expungeStaleEntries(); - SoftReference result = hash.remove(key); - if (result == null) return null; - return result.get(); - } - - public void clear() { - hash.clear(); - reverseLookup.clear(); - } - - public int size() { - expungeStaleEntries(); - return hash.size(); + /** + * Creates a new SoftHashMap with a default retention size size of + * {@link #DEFAULT_RETENTION_SIZE DEFAULT_RETENTION_SIZE} (100 entries). + * + * @see #SoftHashMap(int) + */ + public SoftHashMap() { + this(DEFAULT_RETENTION_SIZE); } /** - * Returns a copy of the key/values in the map at the point of - * calling. However, setValue still sets the value in the - * actual SoftHashMap. + * Creates a new SoftHashMap with the specified retention size. + *

+ * The retention size (n) is the total number of most recent entries in the map that will be strongly referenced + * (ie 'retained') to prevent them from being eagerly garbage collected. That is, the point of a SoftHashMap is to + * allow the garbage collector to remove as many entries from this map as it desires, but there will always be (n) + * elements retained after a GC due to the strong references. + *

+ * Note that in a highly concurrent environments the exact total number of strong references may differ slightly + * than the actual retentionSize value. This number is intended to be a best-effort retention low + * water mark. + * + * @param retentionSize the total number of most recent entries in the map that will be strongly referenced + * (retained), preventing them from being eagerly garbage collected by the JVM. */ - public Set> entrySet() { - expungeStaleEntries(); - Set> result = new LinkedHashSet>(); - for (final Entry> entry : hash.entrySet()) { - final V value = entry.getValue().get(); - if (value != null) { - result.add(new Entry() { - public K getKey() { - return entry.getKey(); - } - public V getValue() { - return value; - } - public V setValue(V v) { - entry.setValue(new SoftReference(v, queue)); - return value; - } - }); + @SuppressWarnings({"unchecked"}) + public SoftHashMap(int retentionSize) { + super(); + RETENTION_SIZE = Math.max(0, retentionSize); + queue = new ReferenceQueue(); + strongReferencesLock = new ReentrantLock(); + map = new ConcurrentHashMap>(); + strongReferences = new ConcurrentLinkedQueue(); + } + + /** + * Creates a {@code SoftHashMap} backed by the specified {@code source}, with a default retention + * size of {@link #DEFAULT_RETENTION_SIZE DEFAULT_RETENTION_SIZE} (100 entries). + * + * @param source the backing map to populate this {@code SoftHashMap} + * @see #SoftHashMap(Map, int) + */ + public SoftHashMap(Map source) { + this(DEFAULT_RETENTION_SIZE); + putAll(source); + } + + /** + * Creates a {@code SoftHashMap} backed by the specified {@code source}, with the specified retention size. + *

+ * The retention size (n) is the total number of most recent entries in the map that will be strongly referenced + * (ie 'retained') to prevent them from being eagerly garbage collected. That is, the point of a SoftHashMap is to + * allow the garbage collector to remove as many entries from this map as it desires, but there will always be (n) + * elements retained after a GC due to the strong references. + *

+ * Note that in a highly concurrent environments the exact total number of strong references may differ slightly + * than the actual retentionSize value. This number is intended to be a best-effort retention low + * water mark. + * + * @param source the backing map to populate this {@code SoftHashMap} + * @param retentionSize the total number of most recent entries in the map that will be strongly referenced + * (retained), preventing them from being eagerly garbage collected by the JVM. + */ + public SoftHashMap(Map source, int retentionSize) { + this(retentionSize); + putAll(source); + } + + public V get(Object key) { + processQueue(); + + V result = null; + SoftValue value = map.get(key); + + if (value != null) { + //unwrap the 'real' value from the SoftReference + result = value.get(); + if (result == null) { + //The wrapped value was garbage collected, so remove this entry from the backing map: + //noinspection SuspiciousMethodCalls + map.remove(key); + } else { + //Add this value to the beginning of the strong reference queue (FIFO). + addToStrongReferences(result); } } return result; } + + private void addToStrongReferences(V result) { + strongReferencesLock.lock(); + try { + strongReferences.add(result); + trimStrongReferencesIfNecessary(); + } finally { + strongReferencesLock.unlock(); + } + + } + + //Guarded by the strongReferencesLock in the addToStrongReferences method + + private void trimStrongReferencesIfNecessary() { + //trim the strong ref queue if necessary: + while (strongReferences.size() > RETENTION_SIZE) { + strongReferences.poll(); + } + } + + /** + * Traverses the ReferenceQueue and removes garbage-collected SoftValue objects from the backing map + * by looking them up using the SoftValue.key data member. + */ + private void processQueue() { + SoftValue sv; + while ((sv = (SoftValue) queue.poll()) != null) { + //noinspection SuspiciousMethodCalls + map.remove(sv.key); // we can access private data! + } + } + + public boolean isEmpty() { + processQueue(); + return map.isEmpty(); + } + + public boolean containsKey(Object key) { + processQueue(); + return map.containsKey(key); + } + + public boolean containsValue(Object value) { + processQueue(); + Collection values = values(); + return values != null && values.contains(value); + } + + public void putAll(Map m) { + if (m == null || m.isEmpty()) { + processQueue(); + return; + } + for (Map.Entry entry : m.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + public Set keySet() { + processQueue(); + return map.keySet(); + } + + public Collection values() { + processQueue(); + Collection keys = map.keySet(); + if (keys.isEmpty()) { + //noinspection unchecked + return Collections.EMPTY_SET; + } + Collection values = new ArrayList(keys.size()); + for (K key : keys) { + V v = get(key); + if (v != null) { + values.add(v); + } + } + return values; + } + + /** + * Creates a new entry, but wraps the value in a SoftValue instance to enable auto garbage collection. + */ + public V put(K key, V value) { + processQueue(); // throw out garbage collected values first + SoftValue sv = new SoftValue(value, key, queue); + SoftValue previous = map.put(key, sv); + addToStrongReferences(value); + return previous != null ? previous.get() : null; + } + + public V remove(Object key) { + processQueue(); // throw out garbage collected values first + SoftValue raw = map.remove(key); + return raw != null ? raw.get() : null; + } + + public void clear() { + strongReferencesLock.lock(); + try { + strongReferences.clear(); + } finally { + strongReferencesLock.unlock(); + } + processQueue(); // throw out garbage collected values + map.clear(); + } + + public int size() { + processQueue(); // throw out garbage collected values first + return map.size(); + } + + public Set> entrySet() { + processQueue(); // throw out garbage collected values first + Collection keys = map.keySet(); + if (keys.isEmpty()) { + //noinspection unchecked + return Collections.EMPTY_SET; + } + + Map kvPairs = new HashMap(keys.size()); + for (K key : keys) { + V v = get(key); + if (v != null) { + kvPairs.put(key, v); + } + } + return kvPairs.entrySet(); + } + + /** + * We define our own subclass of SoftReference which contains + * not only the value but also the key to make it easier to find + * the entry in the HashMap after it's been garbage collected. + */ + private static class SoftValue extends SoftReference { + + private final K key; + + /** + * Constructs a new instance, wrapping the value, key, and queue, as + * required by the superclass. + * + * @param value the map value + * @param key the map key + * @param queue the soft reference queue to poll to determine if the entry had been reaped by the GC. + */ + private SoftValue(V value, K key, ReferenceQueue queue) { + super(value, queue); + this.key = key; + } + + } } \ No newline at end of file diff --git a/core/src/main/java/com/boydti/fawe/object/io/FastByteArrayOutputStream.java b/core/src/main/java/com/boydti/fawe/object/io/FastByteArrayOutputStream.java index 37daec6c..b2b918d3 100644 --- a/core/src/main/java/com/boydti/fawe/object/io/FastByteArrayOutputStream.java +++ b/core/src/main/java/com/boydti/fawe/object/io/FastByteArrayOutputStream.java @@ -41,6 +41,10 @@ public class FastByteArrayOutputStream extends OutputStream { buffer = new byte[blockSize]; } + public FastByteArrayOutputStream(byte[] buffer) { + blockSize = buffer.length; + this.buffer = buffer; + } public int getSize() { return size + index; diff --git a/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweTrim.java b/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweTrim.java new file mode 100644 index 00000000..7261ed69 --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweTrim.java @@ -0,0 +1,57 @@ +package com.boydti.fawe.regions.general.plot; + +import com.boydti.fawe.util.TaskManager; +import com.intellectualcrafters.plot.commands.CommandCategory; +import com.intellectualcrafters.plot.commands.RequiredType; +import com.intellectualcrafters.plot.commands.SubCommand; +import com.intellectualcrafters.plot.config.C; +import com.intellectualcrafters.plot.object.Location; +import com.intellectualcrafters.plot.object.PlotPlayer; +import com.plotsquared.general.commands.CommandDeclaration; +import org.bukkit.Bukkit; + +@CommandDeclaration( + command = "trimchunks", + permission = "plots.admin", + description = "Delete unmodified portions of your plotworld", + usage = "/plot trimchunks ", + requiredType = RequiredType.PLAYER, + category = CommandCategory.ADMINISTRATION) +public class FaweTrim extends SubCommand { + + private boolean ran = false; + + @Override + public boolean onCommand(final PlotPlayer plotPlayer, final String[] strings) { + if (ran) { + plotPlayer.sendMessage("Already running!"); + return false; + } + if (strings.length != 2) { + plotPlayer.sendMessage("First make a backup of your world called then stand in the middle of an empty plot"); + plotPlayer.sendMessage("use /plot trimall "); + return false; + } + if (Bukkit.getWorld(strings[0]) == null) { + C.NOT_VALID_PLOT_WORLD.send(plotPlayer, strings[0]); + return false; + } + ran = true; + TaskManager.IMP.async(new Runnable() { + @Override + public void run() { + try { + PlotTrim trim = new PlotTrim(plotPlayer, plotPlayer.getPlotAreaAbs(), strings[0], Boolean.parseBoolean(strings[1])); + Location loc = plotPlayer.getLocation(); + trim.setChunk(loc.getX() >> 4, loc.getZ() >> 4); + trim.run(); + plotPlayer.sendMessage("Done!"); + } catch (Throwable e) { + e.printStackTrace(); + } + ran = false; + } + }); + return true; + } +} diff --git a/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotSquaredFeature.java b/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotSquaredFeature.java index 4185e6fc..64910141 100644 --- a/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotSquaredFeature.java +++ b/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotSquaredFeature.java @@ -5,6 +5,7 @@ import com.boydti.fawe.object.FawePlayer; import com.boydti.fawe.regions.FaweMask; import com.boydti.fawe.regions.FaweMaskManager; import com.intellectualcrafters.plot.PS; +import com.intellectualcrafters.plot.config.Settings; import com.intellectualcrafters.plot.generator.HybridPlotManager; import com.intellectualcrafters.plot.object.Plot; import com.intellectualcrafters.plot.object.PlotPlayer; @@ -25,6 +26,9 @@ public class PlotSquaredFeature extends FaweMaskManager { setupBlockQueue(); setupSchematicHandler(); setupChunkManager(); + if (Settings.PLATFORM.equals("Bukkit")) { + new FaweTrim(); + } } private void setupBlockQueue() { diff --git a/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotTrim.java b/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotTrim.java new file mode 100644 index 00000000..7c3d64ca --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/regions/general/plot/PlotTrim.java @@ -0,0 +1,219 @@ +package com.boydti.fawe.regions.general.plot; + +import com.boydti.fawe.jnbt.anvil.MCAChunk; +import com.boydti.fawe.jnbt.anvil.MCAFile; +import com.boydti.fawe.jnbt.anvil.MCAFilter; +import com.boydti.fawe.jnbt.anvil.MCAQueue; +import com.boydti.fawe.util.MainUtil; +import com.boydti.fawe.util.MathMan; +import com.intellectualcrafters.plot.PS; +import com.intellectualcrafters.plot.object.ChunkLoc; +import com.intellectualcrafters.plot.object.Location; +import com.intellectualcrafters.plot.object.Plot; +import com.intellectualcrafters.plot.object.PlotArea; +import com.intellectualcrafters.plot.object.PlotPlayer; +import com.intellectualcrafters.plot.util.expiry.ExpireManager; +import com.plotsquared.bukkit.events.PlayerEnterPlotEvent; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.bukkit.Bukkit; +import org.bukkit.event.EventHandler; +import org.bukkit.event.Listener; +import org.bukkit.plugin.Plugin; + + +import static com.google.common.base.Preconditions.checkNotNull; + +public class PlotTrim implements Listener { + private final MCAQueue queue; + private final PlotArea area; + private final PlotPlayer player; + private final MCAQueue originalQueue; + private final File root; + private final File originalRoot; + private byte[][] ids; + private boolean deleteUnowned = true; + + public PlotTrim(PlotPlayer player, PlotArea area, String worldName, boolean deleteUnowned) { + this.root = new File(Bukkit.getWorldContainer(), worldName + "-Copy" + File.separator + "region"); + this.originalRoot = new File(Bukkit.getWorldContainer(), worldName + File.separator + "region"); + this.originalQueue = new MCAQueue(worldName, originalRoot, true); + this.queue = new MCAQueue(worldName + "-Copy", root, true); + this.area = area; + this.player = player; + this.deleteUnowned = deleteUnowned; + } + + public void setChunk(byte[][] ids) { + checkNotNull(ids); + this.ids = ids; + } + + public void setChunk(int x, int z) { + this.ids = originalQueue.getChunk(originalQueue, x, z).ids; + } + + @EventHandler + public void onPlotClaim(PlayerEnterPlotEvent event) { + // Allow deletion + Plot plot = event.getPlot(); + if (plot.getMeta("checkFaweTrim") == null) { + plot.setMeta("checkFaweTrim", true); + removeChunks(plot); + } + } + + private Map chunks = new ConcurrentHashMap<>(); + private Object PRESENT = new Object(); + + private void removeChunks(Plot plot) { + Location pos1 = plot.getBottom(); + Location pos2 = plot.getTop(); + int ccx1 = pos1.getX() >> 4; + int ccz1 = pos1.getZ() >> 4; + int ccx2 = pos2.getX() >> 4; + int ccz2 = pos2.getZ() >> 4; + for (int x = ccx1; x <= ccx2; x++) { + for (int z = ccz1; z <= ccz2; z++) { + long pair = MathMan.pairInt(x, z); + chunks.remove(pair); + } + } + } + + public void run() { + System.out.println("Run!"); + Bukkit.getPluginManager().registerEvents(this, (Plugin) PS.get().IMP); + final Set mcas = new HashSet<>(); + if (deleteUnowned && area != null) { + originalQueue.filterWorld(new MCAFilter() { + @Override + public boolean appliesFile(int mcaX, int mcaZ) { + mcas.add(new ChunkLoc(mcaX, mcaZ)); + return false; + } + }); + ArrayList plots = new ArrayList<>(); + plots.addAll(PS.get().getPlots(area)); + if (ExpireManager.IMP != null) { + plots.removeAll(ExpireManager.IMP.getPendingExpired()); + } + for (Plot plot : plots) { + Location pos1 = plot.getBottom(); + Location pos2 = plot.getTop(); + int ccx1 = pos1.getX() >> 9; + int ccz1 = pos1.getZ() >> 9; + int ccx2 = pos2.getX() >> 9; + int ccz2 = pos2.getZ() >> 9; + for (int x = ccx1; x <= ccx2; x++) { + for (int z = ccz1; z <= ccz2; z++) { + ChunkLoc loc = new ChunkLoc(x, z); + mcas.remove(loc); + } + } + } + for (ChunkLoc mca : mcas) { + int bx = mca.x << 5; + int bz = mca.z << 5; + for (int x = 0; x < 32; x++) { + for (int z = 0; z < 32; z++) { + long pair = MathMan.pairInt(bx + x, bz + z); + chunks.put(pair, PRESENT); + } + } + } + for (Plot plot : plots) { + removeChunks(plot); + } + } + originalQueue.filterWorld(new MCAFilter() { + @Override + public boolean appliesFile(int mcaX, int mcaZ) { + ChunkLoc loc = new ChunkLoc(mcaX, mcaZ); + if (mcas.contains(loc)) { + return false; + } + return true; + } + + @Override + public MCAFile applyFile(MCAFile mca) { + int mcaX = mca.getX(); + int mcaZ = mca.getZ(); + ChunkLoc loc = new ChunkLoc(mcaX, mcaZ); + if (mcas.contains(loc)) { + player.sendMessage("Delete MCA " + mca); + return null; + } + try { + File copy = new File(root, mca.getFile().getName()); + if (!copy.exists()) { + copy = MainUtil.copyFile(mca.getFile(), copy); + player.sendMessage("Filter copy -> " + copy); + } else { + player.sendMessage("Filter existing: " + mcaX + "," + mcaZ); + } + return new MCAFile(mca.getParent(), copy); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + @Override + public boolean appliesChunk(int cx, int cz) { + return true; + } + + @Override + public MCAChunk applyChunk(MCAChunk chunk) { + long pair = MathMan.pairInt(chunk.getX(), chunk.getZ()); + if (chunks.containsKey(pair)) { + chunk.setDeleted(true); + return null; + } + if (ids != null) { + for (int i = 0; i < ids.length; i++) { + if (!isEqual(ids[i], chunk.ids[i])) { + return null; + } + } + chunk.setDeleted(true); + } + return null; + } + }); + player.sendMessage("Done!"); + PlayerEnterPlotEvent.getHandlerList().unregister(this); + } + + private int count = 0; + + private boolean isEqual(byte[] a, byte[] b) { + if (a == b) { + return true; + } + if (a != null) { + if (b != null) { + return Arrays.equals(a, b); + } + return isEmpty(a); + } + return isEmpty(b); + } + + private boolean isEmpty(byte[] a) { + for (byte b : a) { + if (b != 0) { + return false; + } + } + return true; + } +} diff --git a/core/src/main/java/com/boydti/fawe/util/MainUtil.java b/core/src/main/java/com/boydti/fawe/util/MainUtil.java index a73432a9..f67fd193 100644 --- a/core/src/main/java/com/boydti/fawe/util/MainUtil.java +++ b/core/src/main/java/com/boydti/fawe/util/MainUtil.java @@ -39,6 +39,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; @@ -373,6 +374,44 @@ public class MainUtil { } } + public static File copyFile(File sourceFile, File destFile) throws IOException { + if (!destFile.exists()) { + File parent = destFile.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + destFile.createNewFile(); + } + FileInputStream fIn = null; + FileOutputStream fOut = null; + FileChannel source = null; + FileChannel destination = null; + try { + fIn = new FileInputStream(sourceFile); + source = fIn.getChannel(); + fOut = new FileOutputStream(destFile); + destination = fOut.getChannel(); + long transfered = 0; + long bytes = source.size(); + while (transfered < bytes) { + transfered += destination.transferFrom(source, 0, source.size()); + destination.position(transfered); + } + } finally { + if (source != null) { + source.close(); + } else if (fIn != null) { + fIn.close(); + } + if (destination != null) { + destination.close(); + } else if (fOut != null) { + fOut.close(); + } + } + return destFile; + } + public static File copyFile(File jar, String resource, File output) { try { if (output == null) { diff --git a/core/src/main/java/com/boydti/fawe/util/SetQueue.java b/core/src/main/java/com/boydti/fawe/util/SetQueue.java index fb3969ce..6595fb93 100644 --- a/core/src/main/java/com/boydti/fawe/util/SetQueue.java +++ b/core/src/main/java/com/boydti/fawe/util/SetQueue.java @@ -71,7 +71,7 @@ public class SetQueue { if (!MemUtil.isMemoryFree()) { final int mem = MemUtil.calculateMemory(); if (mem != Integer.MAX_VALUE) { - if ((mem <= 1) && Settings.CRASH_MITIGATION) { + if ((mem <= 1) && Settings.PREVENT_CRASHES) { for (FaweQueue queue : getAllQueues()) { queue.saveMemory(); } diff --git a/core/src/main/java/com/sk89q/worldedit/EditSession.java b/core/src/main/java/com/sk89q/worldedit/EditSession.java index e3a3c3bc..c68c5da0 100644 --- a/core/src/main/java/com/sk89q/worldedit/EditSession.java +++ b/core/src/main/java/com/sk89q/worldedit/EditSession.java @@ -271,7 +271,8 @@ public class EditSession extends AbstractWorld implements HasFaweQueue { } else { queue = SetQueue.IMP.getNewQueue(this, fastmode, autoQueue); } - } else if (Settings.EXPERIMENTAL.ANVIL_QUEUE_MODE && !(queue instanceof MCAQueue)) { + } + if (Settings.EXPERIMENTAL.ANVIL_QUEUE_MODE && !(queue instanceof MCAQueue)) { queue = new MCAQueue(queue); } this.queue = queue; diff --git a/core/src/main/java/com/sk89q/worldedit/session/SessionManager.java b/core/src/main/java/com/sk89q/worldedit/session/SessionManager.java index be4a2320..c4cef1fa 100644 --- a/core/src/main/java/com/sk89q/worldedit/session/SessionManager.java +++ b/core/src/main/java/com/sk89q/worldedit/session/SessionManager.java @@ -34,6 +34,8 @@ import com.sk89q.worldedit.util.concurrency.EvenMoreExecutors; import com.sk89q.worldedit.util.eventbus.Subscribe; import java.io.File; import java.io.IOException; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; import java.util.Iterator; import java.util.Map; import java.util.Timer; @@ -63,17 +65,11 @@ public class SessionManager { private final Timer timer = new Timer(); private final WorldEdit worldEdit; private final Map sessions = new ConcurrentHashMap<>(8, 0.9f, 1); - private final Map softSessions = new SoftHashMap<>(); + private final Map> softSessions = new SoftHashMap<>(); private SessionStore store = new VoidStore(); private File path; - // Added // - -// private final ConcurrentLinkedDeque toSave = new ConcurrentLinkedDeque<>(); - /////////// - - /** * Create a new session manager. * @@ -112,11 +108,15 @@ public class SessionManager { return holder.session; } } - Iterator> iter = softSessions.entrySet().iterator(); + Iterator>> iter = softSessions.entrySet().iterator(); while (iter.hasNext()) { - Map.Entry entry = iter.next(); + Map.Entry> entry = iter.next(); UUID key = entry.getKey(); - SessionHolder holder = entry.getValue(); + SessionHolder holder = entry.getValue().get(); + if (holder == null) { + iter.remove(); + continue; + } String test = holder.key.getName(); if (test != null && name.equals(test)) { // if (holder.key.isActive()) { @@ -144,17 +144,19 @@ public class SessionManager { if (stored != null) { return stored.session; } else { - stored = softSessions.get(key); - if (stored != null) { + Reference reference = softSessions.get(key); + if (reference != null) { + stored = reference.get(); + if (stored != null) { // if (stored.key.isActive()) { softSessions.remove(key); sessions.put(key, stored); // } - return stored.session; - } else { - return null; + return stored.session; + } } } + return null; } /** @@ -279,7 +281,7 @@ public class SessionManager { checkNotNull(owner); UUID key = getKey(owner); SessionHolder holder = sessions.remove(key); - softSessions.put(key, holder); + softSessions.put(key, new SoftReference(holder)); save(holder); }