Buffered random access clipboard compression + schematic -> clipboard streaming

Also adds CPUOptimizedClipboard which has no extra compression
Note: Performance degrades if access is actually random (the buffering
optimizes sequential r/w)
Removing stream in favor of clipboard compression / disk
- In order to stream a schematic, it would need to be fully read 3 times
as  tags are not ordered (dimensions -> block ids -> data + tiles +
entities)
- Much faster just using disk / memory as an intermediate step

bukkit110/src/main/resources/plugin.yml

@@ -12,9 +12,6 @@ commands:
     description: (FAWE) Bypass WorldEdit processing and area restrictions
     aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere]
     usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass"
-  stream:
-    description: (FAWE) Stream a schematic into the world
-    aliases: [/stream]
   fawe:
     description: (FAWE) Reload the plugin
     aliases: [/fawe,/fawereload]

bukkit1710/src/main/resources/plugin.yml

@@ -12,9 +12,6 @@ commands:
     description: (FAWE) Bypass WorldEdit processing and area restrictions
     aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere]
     usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass"
-  stream:
-    description: (FAWE) Stream a schematic into the world
-    aliases: [/stream]
   fawe:
     description: (FAWE) Reload the plugin
     aliases: [/fawe,/fawereload]

bukkit18/src/main/resources/plugin.yml

@@ -12,9 +12,6 @@ commands:
     description: (FAWE) Bypass WorldEdit processing and area restrictions
     aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere]
     usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass"
-  stream:
-    description: (FAWE) Stream a schematic into the world
-    aliases: [/stream]
   fawe:
     description: (FAWE) Reload the plugin
     aliases: [/fawe,/fawereload]

bukkit19/src/main/resources/plugin.yml

@@ -15,9 +15,6 @@ commands:
   fixlighting:
     description: (FAWE) Fix the lighting in your current chunk
     aliases: [/fixlighting]
-  stream:
-    description: (FAWE) Stream a schematic into the world
-    aliases: [/stream]
   fawe:
     description: (FAWE) Reload the plugin
     aliases: [/fawe,/fawereload]

core/src/main/java/com/boydti/fawe/Fawe.java

@@ -2,7 +2,6 @@ package com.boydti.fawe;
 
 import com.boydti.fawe.command.Cancel;
 import com.boydti.fawe.command.Reload;
-import com.boydti.fawe.command.Stream;
 import com.boydti.fawe.command.Wea;
 import com.boydti.fawe.command.WorldEditRegion;
 import com.boydti.fawe.config.BBC;
@@ -68,6 +67,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 import javax.management.InstanceAlreadyExistsException;
 import javax.management.Notification;
 import javax.management.NotificationEmitter;
@@ -226,7 +226,6 @@ public class Fawe {
 
     private void setupCommands() {
         this.IMP.setupCommand("wea", new Wea());
-        this.IMP.setupCommand("stream", new Stream());
         this.IMP.setupCommand("select", new WorldEditRegion());
         this.IMP.setupCommand("fawe", new Reload());
         this.IMP.setupCommand("fcancel", new Cancel());

core/src/main/java/com/boydti/fawe/command/Stream.java

@@ -1,37 +0,0 @@
-package com.boydti.fawe.command;
-
-import com.boydti.fawe.Fawe;
-import com.boydti.fawe.FaweAPI;
-import com.boydti.fawe.config.BBC;
-import com.boydti.fawe.object.FaweCommand;
-import com.boydti.fawe.object.FawePlayer;
-import java.io.File;
-
-public class Stream extends FaweCommand {
-
-    public Stream() {
-        super("fawe.stream");
-    }
-
-    @Override
-    public boolean execute(final FawePlayer player, final String... args) {
-        if (player == null) {
-            return false;
-        }
-        if (args.length != 1) {
-            BBC.COMMAND_SYNTAX.send(player, "/stream <file>");
-            return false;
-        }
-        if (!args[0].endsWith(".schematic")) {
-            args[0] += ".schematic";
-        }
-        final File file = Fawe.get().getWorldEdit().getWorkingDirectoryFile(Fawe.get().getWorldEdit().getConfiguration().saveDir + File.separator + args[0]);
-        if (!file.exists()) {
-            BBC.SCHEMATIC_NOT_FOUND.send(player, args[0]);
-            return false;
-        }
-        FaweAPI.streamSchematic(file, player.getLocation());
-        BBC.SCHEMATIC_PASTING.send(player);
-        return true;
-    }
-}

core/src/main/java/com/boydti/fawe/config/Settings.java

@@ -201,8 +201,17 @@ public class Settings extends Config {
     }
 
     public static class CLIPBOARD {
+        @Comment("Store the clipboard on disk instead of memory")
         public static boolean USE_DISK = false;
-
+        @Comment({
+                "Compress the clipboard to reduce the size:",
+                " - TODO: Buffered random access with compression is not implemented on disk yet",
+                " - 0 = No compression",
+                " - 1 = Fast compression",
+                " - 2-17 = Slower compression"
+        })
+        public static int COMPRESSION_LEVEL = 1;
+        @Comment("Number of days to keep history on disk before deleting it")
         public static int DELETE_AFTER_DAYS = 1;
     }
 

core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java

@@ -46,4 +46,13 @@ public class NBTStreamer {
             return node;
         }
     }
+
+    public static abstract class ByteReader extends RunnableVal2<Integer, Integer> {
+        @Override
+        public void run(Integer index, Integer value) {
+            run(index, value);
+        }
+
+        public abstract void run(int index, int byteValue);
+    }
 }

core/src/main/java/com/boydti/fawe/jnbt/SchematicStreamer.java

@@ -1,7 +1,11 @@
 package com.boydti.fawe.jnbt;
 
+import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.object.RunnableVal2;
+import com.boydti.fawe.object.clipboard.CPUOptimizedClipboard;
 import com.boydti.fawe.object.clipboard.DiskOptimizedClipboard;
+import com.boydti.fawe.object.clipboard.FaweClipboard;
+import com.boydti.fawe.object.clipboard.MemoryOptimizedClipboard;
 import com.sk89q.jnbt.CompoundTag;
 import com.sk89q.jnbt.ListTag;
 import com.sk89q.jnbt.NBTInputStream;
@@ -12,76 +16,19 @@ import com.sk89q.worldedit.extent.clipboard.Clipboard;
 import com.sk89q.worldedit.regions.CuboidRegion;
 import java.io.IOException;
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
 
 public class SchematicStreamer extends NBTStreamer {
     private final UUID uuid;
 
-    public SchematicStreamer(NBTInputStream stream, UUID uuid) throws IOException {
+    public SchematicStreamer(NBTInputStream stream, UUID uuid) {
         super(stream);
         this.uuid = uuid;
-        addReader("Schematic.Height", new RunnableVal2<Integer, Short>() {
+        clipboard = new BlockArrayClipboard(new CuboidRegion(new Vector(0, 0, 0), new Vector(0, 0, 0)), fc);
-            @Override
+    }
-            public void run(Integer index, Short value) {
+
-                height = (value);
+    public void addBlockReaders() {
-            }
+        final long start = System.currentTimeMillis();
-        });
+        NBTStreamReader initializer = new NBTStreamReader<Integer, Integer>() {
-        addReader("Schematic.Width", new RunnableVal2<Integer, Short>() {
-            @Override
-            public void run(Integer index, Short value) {
-                width = (value);
-            }
-        });
-        addReader("Schematic.Length", new RunnableVal2<Integer, Short>() {
-            @Override
-            public void run(Integer index, Short value) {
-                length = (value);
-            }
-        });
-        final AtomicInteger originX = new AtomicInteger();
-        final AtomicInteger originY = new AtomicInteger();
-        final AtomicInteger originZ = new AtomicInteger();
-        addReader("Schematic.WEOriginX", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                originX.set(value);
-            }
-        });
-        addReader("Schematic.WEOriginY", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                originY.set(value);
-            }
-        });
-        addReader("Schematic.WEOriginZ", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                originZ.set(value);
-            }
-        });
-        final AtomicInteger offsetX = new AtomicInteger();
-        final AtomicInteger offsetY = new AtomicInteger();
-        final AtomicInteger offsetZ = new AtomicInteger();
-        addReader("Schematic.WEOffsetX", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                offsetX.set(value);
-            }
-        });
-        addReader("Schematic.WEOffsetY", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                offsetY.set(value);
-            }
-        });
-        addReader("Schematic.WEOffsetZ", new RunnableVal2<Integer, Integer>() {
-            @Override
-            public void run(Integer index, Integer value) {
-                offsetZ.set(value);
-            }
-        });
-        // Blocks
-        RunnableVal2<Integer, Integer> initializer = new RunnableVal2<Integer, Integer>() {
             @Override
             public void run(Integer length, Integer type) {
                 setupClipboard(length);
@@ -90,27 +37,25 @@ public class SchematicStreamer extends NBTStreamer {
         addReader("Schematic.Blocks.?", initializer);
         addReader("Schematic.Data.?", initializer);
         addReader("Schematic.AddBlocks.?", initializer);
-        addReader("Schematic.Blocks.#", new RunnableVal2<Integer, Byte>() {
+        addReader("Schematic.Blocks.#", new ByteReader() {
-            int i;
             @Override
-            public void run(Integer index, Byte value) {
+            public void run(int index, int value) {
-                fc.setId(i++, value);
+                fc.setId(index, value);
             }
         });
-        addReader("Schematic.Data.#", new RunnableVal2<Integer, Byte>() {
+        addReader("Schematic.Data.#", new ByteReader() {
-            int i;
             @Override
-            public void run(Integer index, Byte value) {
+            public void run(int index, int value) {
-                fc.setData(i++, value);
+                fc.setData(index, value);
             }
         });
-        addReader("Schematic.AddBlocks.#", new RunnableVal2<Integer, Byte>() {
+        addReader("Schematic.AddBlocks.#", new ByteReader() {
-            int i;
             @Override
-            public void run(Integer index, Byte value) {
+            public void run(int index, int value) {
-                fc.setAdd(i++, value);
+                fc.setAdd(index, value);
             }
         });
+
         // Tiles
         addReader("Schematic.TileEntities.#", new RunnableVal2<Integer, CompoundTag>() {
             @Override
@@ -141,35 +86,124 @@ public class SchematicStreamer extends NBTStreamer {
                 fc.createEntity(null, positionTag.asDouble(0), positionTag.asDouble(1), positionTag.asDouble(2), (float) directionTag.asDouble(0), (float) directionTag.asDouble(1), state);
             }
         });
-        readFully();
+    }
-        Vector min = new Vector(originX.get(), originY.get(), originZ.get());
+
-        Vector offset = new Vector(offsetX.get(), offsetY.get(), offsetZ.get());
+    public void addDimensionReaders() {
-        Vector origin = min.subtract(offset);
+        addReader("Schematic.Height", new RunnableVal2<Integer, Short>() {
-        Vector dimensions = new Vector(width, height, length);
+            @Override
-        fc.setDimensions(dimensions);
+            public void run(Integer index, Short value) {
-        CuboidRegion region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE));
+                height = (value);
-        clipboard = new BlockArrayClipboard(region, fc);
+            }
-        clipboard.setOrigin(origin);
+        });
+        addReader("Schematic.Width", new RunnableVal2<Integer, Short>() {
+            @Override
+            public void run(Integer index, Short value) {
+                width = (value);
+            }
+        });
+        addReader("Schematic.Length", new RunnableVal2<Integer, Short>() {
+            @Override
+            public void run(Integer index, Short value) {
+                length = (value);
+            }
+        });
+        addReader("Schematic.WEOriginX", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                originX = (value);
+            }
+        });
+        addReader("Schematic.WEOriginY", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                originY = (value);
+            }
+        });
+        addReader("Schematic.WEOriginZ", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                originZ = (value);
+            }
+        });
+        addReader("Schematic.WEOffsetX", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                offsetX = (value);
+            }
+        });
+        addReader("Schematic.WEOffsetY", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                offsetY = (value);
+            }
+        });
+        addReader("Schematic.WEOffsetZ", new RunnableVal2<Integer, Integer>() {
+            @Override
+            public void run(Integer index, Integer value) {
+                offsetZ = (value);
+            }
+        });
     }
 
     private int height;
     private int width;
     private int length;
 
-    private Clipboard clipboard;
+    private int originX;
-    private DiskOptimizedClipboard fc;
+    private int originY;
+    private int originZ;
 
-    private DiskOptimizedClipboard setupClipboard(int size) {
+    private int offsetX;
+    private int offsetY;
+    private int offsetZ;
+
+    private BlockArrayClipboard clipboard;
+    private FaweClipboard fc;
+
+    private FaweClipboard setupClipboard(int size) {
         if (fc != null) {
             if (fc.getDimensions().getX() == 0) {
                 fc.setDimensions(new Vector(size, 1, 1));
             }
             return fc;
         }
-        return fc = new DiskOptimizedClipboard(size, 1, 1, uuid);
+        if (Settings.CLIPBOARD.USE_DISK) {
+            return fc = new DiskOptimizedClipboard(size, 1, 1, uuid);
+        } else if (Settings.CLIPBOARD.COMPRESSION_LEVEL == 0) {
+            return fc = new CPUOptimizedClipboard(size, 1, 1);
+        } else {
+            return fc = new MemoryOptimizedClipboard(size, 1, 1);
+        }
     }
 
-    public Clipboard getClipboard() {
+    public Vector getOrigin() {
+        return new Vector(originX, originY, originZ);
+    }
+
+    public Vector getOffset() {
+        return new Vector(offsetX, offsetY, offsetZ);
+    }
+
+    public Vector getDimensions() {
+        return new Vector(width, height, length);
+    }
+
+    public void setClipboard(FaweClipboard clipboard) {
+        this.fc = clipboard;
+    }
+
+    public Clipboard getClipboard() throws IOException {
+        addDimensionReaders();
+        addBlockReaders();
+        readFully();
+        Vector min = new Vector(originX, originY, originZ);
+        Vector offset = new Vector(offsetX, offsetY, offsetZ);
+        Vector origin = min.subtract(offset);
+        Vector dimensions = new Vector(width, height, length);
+        fc.setDimensions(dimensions);
+        CuboidRegion region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE));
+        clipboard.init(region, fc);
+        clipboard.setOrigin(origin);
         return clipboard;
     }
 }

core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java

@@ -0,0 +1,177 @@
+package com.boydti.fawe.object.clipboard;
+
+import com.boydti.fawe.FaweCache;
+import com.boydti.fawe.object.RunnableVal2;
+import com.sk89q.jnbt.CompoundTag;
+import com.sk89q.worldedit.Vector;
+import com.sk89q.worldedit.blocks.BaseBlock;
+import com.sk89q.worldedit.entity.BaseEntity;
+import com.sk89q.worldedit.entity.Entity;
+import com.sk89q.worldedit.extent.Extent;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+public class CPUOptimizedClipboard extends FaweClipboard {
+    public CPUOptimizedClipboard(int width, int height, int length) {
+        this.width = width;
+        this.height = height;
+        this.length = length;
+        this.area = width * length;
+        this.volume = area * height;
+        ids = new byte[volume];
+        datas = new byte[volume];
+        nbtMap = new HashMap<>();
+        entities = new HashSet<>();
+    }
+
+    private int length;
+    private int height;
+    private int width;
+    private int area;
+    private int volume;
+    private byte[] ids;
+    private byte[] datas;
+    private byte[] add;
+    private final HashMap<Integer, CompoundTag> nbtMap;
+    private final HashSet<ClipboardEntity> entities;
+
+    public int getId(int index) {
+        if (add != null) {
+            return ids[index] & 0xFF + add[index] & 0xFF;
+        }
+        return ids[index] & 0xFF;
+    }
+
+    public int getData(int index) {
+        return datas[index];
+    }
+
+    @Override
+    public void setDimensions(Vector dimensions) {
+        width = dimensions.getBlockX();
+        height = dimensions.getBlockY();
+        length = dimensions.getBlockZ();
+        area = width * length;
+    }
+
+    @Override
+    public Vector getDimensions() {
+        return new Vector(width, height, length);
+    }
+
+    @Override
+    public void setAdd(int index, int value) {
+        if (value == 0) {
+            return;
+        }
+        if (this.add == null) {
+            add = new byte[volume];
+        }
+        add[index] = (byte) value;
+    }
+
+    @Override
+    public void setId(int index, int value) {
+        ids[index] = (byte) value;
+    }
+
+    @Override
+    public void setData(int index, int value) {
+        datas[index] = (byte) value;
+    }
+
+    private int ylast;
+    private int ylasti;
+    private int zlast;
+    private int zlasti;
+
+    public int getIndex(int x, int y, int z) {
+        return x + ((ylast == y) ? ylasti : (ylasti = (ylast = y) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
+    }
+
+    @Override
+    public BaseBlock getBlock(int x, int y, int z) {
+        int index = getIndex(x, y, z);
+        return getBlock(index);
+    }
+
+    public BaseBlock getBlock(int index) {
+        int id = getId(index);
+        if (id == 0) {
+            return FaweCache.CACHE_BLOCK[0];
+        }
+        BaseBlock block;
+        if (FaweCache.hasData(id)) {
+            block = FaweCache.getBlock(id, getData(index));
+        } else {
+            block = FaweCache.getBlock(id, 0);
+        }
+        if (FaweCache.hasNBT(id)) {
+            CompoundTag nbt = nbtMap.get(index);
+            if (nbt != null) {
+                block = new BaseBlock(block.getId(), block.getData());
+                block.setNbtData(nbt);
+            }
+        }
+        return block;
+    }
+
+    @Override
+    public void forEach(final RunnableVal2<Vector,BaseBlock> task, boolean air) {
+        task.value1 = new Vector(0, 0, 0);
+        for (int y = 0, index = 0; y < height; y++) {
+            for (int z = 0; z < length; z++) {
+                for (int x = 0; x < width; x++, index++) {
+                    task.value2 = getBlock(index);
+                    if (!air && task.value2.getId() == 0) {
+                        continue;
+                    }
+                    task.value1.x = x;
+                    task.value1.y = y;
+                    task.value1.z = z;
+                    task.run();
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean setTile(int x, int y, int z, CompoundTag tag) {
+        nbtMap.put(getIndex(x, y, z), tag);
+        return true;
+    }
+
+    @Override
+    public boolean setBlock(int x, int y, int z, BaseBlock block) {
+        return setBlock(getIndex(x, y, z), block);
+    }
+
+    public boolean setBlock(int index, BaseBlock block) {
+        setId(index, (byte) block.getId());
+        setData(index, (byte) block.getData());
+        CompoundTag tile = block.getNbtData();
+        if (tile != null) {
+            nbtMap.put(index, tile);
+        }
+        return true;
+    }
+
+    @Override
+    public Entity createEntity(Extent world, double x, double y, double z, float yaw, float pitch, BaseEntity entity) {
+        FaweClipboard.ClipboardEntity ret = new ClipboardEntity(world, x, y, z, yaw, pitch, entity);
+        entities.add(ret);
+        return ret;
+    }
+
+    @Override
+    public List<? extends Entity> getEntities() {
+        return new ArrayList<>(entities);
+    }
+
+    @Override
+    public boolean remove(ClipboardEntity clipboardEntity) {
+        return entities.remove(clipboardEntity);
+    }
+}

core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java

@@ -76,6 +76,7 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
         autoCloseTask();
     }
 
+    @Override
     public Vector getDimensions() {
         return new Vector(width, height, length);
     }
@@ -141,6 +142,7 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
         }
     }
 
+    @Override
     public void setDimensions(Vector dimensions) {
         try {
             if (raf == null) {
@@ -331,7 +333,8 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
         return false;
     }
 
-    public boolean setId(int i, int id) {
+    @Override
+    public void setId(int i, int id) {
         try {
             if (raf == null) {
                 open();
@@ -344,14 +347,12 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
             int combined = FaweCache.getData(raf.readChar()) + (id << 4);
             raf.seekUnsafe(raf.getFilePointer() - 2);
             raf.writeChar(combined);
-            return true;
         }  catch (Exception e) {
             MainUtil.handleError(e);
         }
-        return false;
     }
 
-    public boolean setCombined(int i, int combined) {
+    public void setCombined(int i, int combined) {
         try {
             if (raf == null) {
                 open();
@@ -362,14 +363,13 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
             }
             last = i;
             raf.writeChar(combined);
-            return true;
         }  catch (Exception e) {
             MainUtil.handleError(e);
         }
-        return false;
     }
 
-    public boolean setAdd(int i, int add) {
+    @Override
+    public void setAdd(int i, int add) {
         try {
             if (raf == null) {
                 open();
@@ -382,31 +382,13 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
             int combined = raf.readChar() + (add << 4);
             raf.seekUnsafe(raf.getFilePointer() - 2);
             raf.writeChar(combined);
-            return true;
         }  catch (Exception e) {
             MainUtil.handleError(e);
         }
-        return false;
     }
 
-    public int getCombined(int i) {
+    @Override
-        try {
+    public void setData(int i, int data) {
-            if (raf == null) {
-                open();
-            }
-            if (i != last + 1) {
-                raf.seek((HEADER_SIZE) + (i << 1));
-                lastAccessed = System.currentTimeMillis();
-            }
-            last = i;
-            return raf.readChar();
-        }  catch (Exception e) {
-            MainUtil.handleError(e);
-        }
-        return 0;
-    }
-
-    public boolean setData(int i, int data) {
         try {
             if (raf == null) {
                 open();
@@ -419,11 +401,9 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable {
             int combined = (FaweCache.getId(raf.readChar()) << 4) + data;
             raf.seekUnsafe(raf.getFilePointer() - 2);
             raf.writeChar(combined);
-            return true;
         }  catch (Exception e) {
             MainUtil.handleError(e);
         }
-        return false;
     }
 
     @Override

core/src/main/java/com/boydti/fawe/object/clipboard/FaweClipboard.java

@@ -19,6 +19,12 @@ public abstract class FaweClipboard {
 
     public abstract boolean setBlock(int x, int y, int z, BaseBlock block);
 
+    public abstract void setId(int index, int id);
+
+    public abstract void setData(int index, int data);
+
+    public abstract void setAdd(int index, int id);
+
     public abstract boolean setTile(int x, int y, int z, CompoundTag tag);
 
     public abstract Entity createEntity(Extent world, double x, double y, double z, float yaw, float pitch, BaseEntity entity);
@@ -29,6 +35,10 @@ public abstract class FaweClipboard {
 
     public void setOrigin(Vector offset) {} // Do nothing
 
+    public abstract void setDimensions(Vector dimensions);
+
+    public abstract Vector getDimensions();
+
     /**
      * The locations provided are relative to the clipboard min
      * @param task

core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java

@@ -1,11 +1,10 @@
 package com.boydti.fawe.object.clipboard;
 
 import com.boydti.fawe.FaweCache;
-import com.boydti.fawe.object.IntegerTrio;
+import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.object.RunnableVal2;
+import com.boydti.fawe.util.MainUtil;
 import com.sk89q.jnbt.CompoundTag;
-import com.sk89q.worldedit.BlockVector;
-import com.sk89q.worldedit.EditSession;
 import com.sk89q.worldedit.Vector;
 import com.sk89q.worldedit.blocks.BaseBlock;
 import com.sk89q.worldedit.entity.BaseEntity;
@@ -17,30 +16,239 @@ import java.util.HashSet;
 import java.util.List;
 
 public class MemoryOptimizedClipboard extends FaweClipboard {
-    protected int length;
-    protected int height;
-    protected int width;
-    protected int area;
 
-    // x,z,y+15>>4 | y&15
+    public static final int BLOCK_SIZE = 1048576;
-    private final byte[][] ids;
+    public static final int BLOCK_MASK = 1048575;
-    private final byte[] heights;
+    public static final int BLOCK_SHIFT = 20;
+
+    private int length;
+    private int height;
+    private int width;
+    private int area;
+    private int volume;
+
+    private byte[][] ids;
     private byte[][] datas;
-    private final HashMap<IntegerTrio, CompoundTag> nbtMap;
+    private byte[][] add;
+
+    private byte[] buffer = new byte[MainUtil.getMaxCompressedLength(BLOCK_SIZE)];
+
+    private final HashMap<Integer, CompoundTag> nbtMap;
     private final HashSet<ClipboardEntity> entities;
 
+    private int lastIdsI = -1;
+    private int lastDatasI = -1;
+    private int lastAddI = -1;
+
+    private byte[] lastIds;
+    private byte[] lastDatas;
+    private byte[] lastAdd;
+
+    private boolean saveIds = false;
+    private boolean saveDatas = false;
+    private boolean saveAdd = false;
+    
+    private int compressionLevel;
+
     public MemoryOptimizedClipboard(int width, int height, int length) {
+        this(width, height, length, Settings.CLIPBOARD.COMPRESSION_LEVEL);
+    }
+
+    public MemoryOptimizedClipboard(int width, int height, int length, int compressionLevel) {
         this.width = width;
         this.height = height;
         this.length = length;
         this.area = width * length;
-        heights = new byte[(height + 15) >>  4];
+        this.volume = area * height;
-        for (int y = 0; y < ((height + 15) >>  4); y++) {
+        ids = new byte[1 + (volume >> BLOCK_SHIFT)][];
-            heights[y] = (byte) Math.min(16, height - (y << 4));
+        datas = new byte[1 + (volume >> BLOCK_SHIFT)][];
-        }
-        ids = new byte[width * length * ((height + 15) >>  4)][];
         nbtMap = new HashMap<>();
         entities = new HashSet<>();
+        this.compressionLevel = compressionLevel;
+    }
+
+    public int getId(int index) {
+        int i = index >> BLOCK_SHIFT;
+        if (i == lastIdsI) {
+            if (lastIds == null) {
+                return 0;
+            }
+            if (add == null) {
+                return lastIds[index & BLOCK_MASK] & 0xFF;
+            } else {
+                return lastIds[index & BLOCK_MASK] & 0xFF + getAdd(index);
+            }
+        }
+        saveIds();
+        byte[] compressed = ids[lastIdsI = i];
+        if (compressed == null) {
+            lastIds = null;
+            return 0;
+        }
+        lastIds = MainUtil.decompress(compressed, lastIds, BLOCK_SIZE, compressionLevel);
+        if (add == null) {
+            return lastIds[index & BLOCK_MASK] & 0xFF;
+        } else {
+            return lastIds[index & BLOCK_MASK] & 0xFF + getAdd(index);
+        }
+    }
+
+    int saves = 0;
+
+    private void saveIds() {
+        if (saveIds && lastIds != null) {
+            ids[lastIdsI] = MainUtil.compress(lastIds, buffer, compressionLevel);
+        }
+        saveIds = false;
+    }
+
+    private void saveDatas() {
+        if (saveDatas && lastDatas != null) {
+            datas[lastDatasI] = MainUtil.compress(lastDatas, buffer, compressionLevel);
+        }
+        saveDatas = false;
+    }
+
+    private void saveAdd() {
+        if (saveAdd && lastAdd != null) {
+            add[lastAddI] = MainUtil.compress(lastAdd, buffer, compressionLevel);
+        }
+        saveAdd = false;
+    }
+
+    public int getData(int index) {
+        int i = index >> BLOCK_SHIFT;
+        if (i == lastDatasI) {
+            if (lastDatas == null) {
+                return 0;
+            }
+            return lastDatas[index & BLOCK_MASK];
+        }
+        saveDatas();
+        byte[] compressed = datas[lastDatasI = i];
+        if (compressed == null) {
+            lastDatas = null;
+            return 0;
+        }
+        lastDatas = MainUtil.decompress(compressed, lastDatas, BLOCK_SIZE, compressionLevel);
+        return lastDatas[index & BLOCK_MASK];
+    }
+
+    @Override
+    public void setDimensions(Vector dimensions) {
+        width = dimensions.getBlockX();
+        height = dimensions.getBlockY();
+        length = dimensions.getBlockZ();
+        area = width * length;
+    }
+
+    @Override
+    public Vector getDimensions() {
+        return new Vector(width, height, length);
+    }
+
+    public int getAdd(int index) {
+        int i = index >> BLOCK_SHIFT;
+        if (i == lastAddI) {
+            if (lastAdd == null) {
+                return 0;
+            }
+            return lastAdd[index & BLOCK_MASK] & 0xFF;
+        }
+        saveAdd();
+        byte[] compressed = add[lastAddI = i];
+        if (compressed == null) {
+            lastAdd = null;
+            return 0;
+        }
+        lastAdd = MainUtil.decompress(compressed, lastAdd, BLOCK_SIZE, compressionLevel);
+        return lastAdd[index & BLOCK_MASK] & 0xFF;
+    }
+
+
+    private int lastI;
+    private int lastIMin;
+    private int lastIMax;
+
+    public int getLocalIndex(int index) {
+        if (index < lastIMin || index > lastIMax) {
+            lastI = index >> BLOCK_SHIFT;
+            lastIMin = lastI << BLOCK_SHIFT;
+            lastIMax = lastIMin + BLOCK_MASK;
+        }
+        return lastI;
+    }
+
+    @Override
+    public void setId(int index, int value) {
+        int i = getLocalIndex(index);
+        if (i != lastIdsI) {
+            saveIds();
+            byte[] compressed = ids[lastIdsI = i];
+            if (compressed != null) {
+                lastIds = MainUtil.decompress(compressed, lastIds, BLOCK_SIZE, compressionLevel);
+            } else {
+                lastIds = null;
+            }
+        }
+        if (lastIds == null) {
+            if (value == 0) {
+                return;
+            }
+            lastIds = new byte[BLOCK_SIZE];
+        }
+        lastIds[index & BLOCK_MASK] = (byte) value;
+        saveIds = true;
+    }
+
+    @Override
+    public void setData(int index, int value) {
+        int i = getLocalIndex(index);
+        if (i != lastDatasI) {
+            saveDatas();
+            byte[] compressed = datas[lastDatasI = i];
+            if (compressed != null) {
+                lastDatas = MainUtil.decompress(compressed, lastDatas, BLOCK_SIZE, compressionLevel);
+            } else {
+                lastDatas = null;
+            }
+        }
+        if (lastDatas == null) {
+            if (value == 0) {
+                return;
+            }
+            lastDatas = new byte[BLOCK_SIZE];
+        }
+        lastDatas[index & BLOCK_MASK] = (byte) value;
+        saveDatas = true;
+    }
+
+    @Override
+    public void setAdd(int index, int value) {
+        if (value == 0) {
+            return;
+        }
+        if (add == null) {
+            add = new byte[1 + (volume >> BLOCK_SHIFT)][];
+        }
+        int i = index >> BLOCK_SHIFT;
+        if (i != lastAddI) {
+            saveAdd();
+            byte[] compressed = add[lastAddI = i];
+            if (compressed != null) {
+                lastAdd = MainUtil.decompress(compressed, lastAdd, BLOCK_SIZE, compressionLevel);
+            } else {
+                lastAdd = null;
+            }
+        }
+        if (lastAdd == null) {
+            if (value == 0) {
+                return;
+            }
+            lastAdd = new byte[BLOCK_SIZE];
+        }
+        lastAdd[index & BLOCK_MASK] = (byte) value;
+        saveAdd = true;
     }
 
     private int ylast;
@@ -48,28 +256,29 @@ public class MemoryOptimizedClipboard extends FaweClipboard {
     private int zlast;
     private int zlasti;
 
+    public int getIndex(int x, int y, int z) {
+        return x + ((ylast == y) ? ylasti : (ylasti = (ylast = y) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
+    }
+
     @Override
     public BaseBlock getBlock(int x, int y, int z) {
-        int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
+        int index = getIndex(x, y, z);
-        byte[] idArray = ids[i];
+        return getBlock(index);
-        if (idArray == null) {
+    }
+
+    public BaseBlock getBlock(int index) {
+        int id = getId(index);
+        if (id == 0) {
             return FaweCache.CACHE_BLOCK[0];
         }
-        int y2 = y & 0xF;
-        int id = idArray[y2] & 0xFF;
         BaseBlock block;
-        if (!FaweCache.hasData(id) || datas == null) {
+        if (FaweCache.hasData(id)) {
-            block = FaweCache.CACHE_BLOCK[id << 4];
+            block = FaweCache.getBlock(id, getData(index));
         } else {
-            byte[] dataArray = datas[i];
+            block = FaweCache.getBlock(id, 0);
-            if (dataArray == null) {
-                block = FaweCache.CACHE_BLOCK[id << 4];
-            } else {
-                block = FaweCache.CACHE_BLOCK[(id << 4) + dataArray[y2]];
-            }
         }
         if (FaweCache.hasNBT(id)) {
-            CompoundTag nbt = nbtMap.get(new IntegerTrio(x, y, z));
+            CompoundTag nbt = nbtMap.get(index);
             if (nbt != null) {
                 block = new BaseBlock(block.getId(), block.getData());
                 block.setNbtData(nbt);
@@ -80,244 +289,60 @@ public class MemoryOptimizedClipboard extends FaweClipboard {
 
     @Override
     public void forEach(final RunnableVal2<Vector,BaseBlock> task, boolean air) {
-        BlockVector pos = new BlockVector(0, 0, 0);
+//        Fawe.debug("Compressed: " + size() + "b | Uncompressed: " + (volume << 0x5) + "b");
-        int y1max = ((height + 15) >>  4);
+        task.value1 = new Vector(0, 0, 0);
-        for (int x = 0; x < width; x++) {
+        for (int y = 0, index = 0; y < height; y++) {
-            int i1 = x;
             for (int z = 0; z < length; z++) {
-                int i2 = i1 + z * width;
+                for (int x = 0; x < width; x++, index++) {
-                for (int y = 0; y < y1max; y++) {
+                    task.value2 = getBlock(index);
-                    int y1 = y << 4;
+                    if (!air && task.value2.getId() == 0) {
-                    int i = i2 + y * area;
-                    byte[] idArray = ids[i];
-                    if (idArray == null) {
-                        if (!air) {
-                            continue;
-                        }
-                        for (int y2 = 0; y2 < height; y2++) {
-                            pos.x = x;
-                            pos.z = z;
-                            pos.y = y1 + y2;
-                            int yy = y1 + y2;
-                            if (yy >= height) {
-                                break;
-                            }
-                            task.run(pos, EditSession.nullBlock);
-                        }
                         continue;
                     }
-                    for (int y2 = 0; y2 < idArray.length; y2++) {
+                    task.value1.x = x;
-                        int yy = y1 + y2;
+                    task.value1.y = y;
-                        if (yy >= height) {
+                    task.value1.z = z;
-                            break;
+                    task.run();
-                        }
-                        int id = idArray[y2] & 0xFF;
-                        if (id == 0 && !air) {
-                            continue;
-                        }
-                        pos.x = x;
-                        pos.z = z;
-                        pos.y = y1 + y2;
-                        BaseBlock block;
-                        if (!FaweCache.hasData(id) || datas == null) {
-                            block = FaweCache.CACHE_BLOCK[id << 4];
-                        } else {
-                            byte[] dataArray = datas[i];
-                            if (dataArray == null) {
-                                block = FaweCache.CACHE_BLOCK[id << 4];
-                            } else {
-                                block = FaweCache.CACHE_BLOCK[(id << 4) + dataArray[y2]];
-                            }
-                        }
-                        if (FaweCache.hasNBT(id)) {
-                            CompoundTag nbt = nbtMap.get(new IntegerTrio((int) pos.x, (int) pos.y, (int) pos.z));
-                            if (nbt != null) {
-                                block = new BaseBlock(block.getId(), block.getData());
-                                block.setNbtData(nbt);
-                            }
-                        }
-                        task.run(pos, block);
-                    }
                 }
             }
         }
     }
 
+    public int size() {
+        saveIds();
+        saveDatas();
+        int total = 0;
+        for (byte[] array : ids) {
+            if (array != null) {
+                total += array.length;
+            }
+        }
+        for (byte[] array : datas) {
+            if (array != null) {
+                total += array.length;
+            }
+        }
+        return total;
+    }
+
     @Override
     public boolean setTile(int x, int y, int z, CompoundTag tag) {
-        nbtMap.put(new IntegerTrio(x, y, z), tag);
+        nbtMap.put(getIndex(x, y, z), tag);
         return true;
     }
 
     @Override
     public boolean setBlock(int x, int y, int z, BaseBlock block) {
-        final int id = block.getId();
+        return setBlock(getIndex(x, y, z), block);
-        switch (id) {
+    }
-            case 0: {
+
-                int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
+    public boolean setBlock(int index, BaseBlock block) {
-                byte[] idArray = ids[i];
+        setId(index, (byte) block.getId());
-                if (idArray != null) {
+        setData(index, (byte) block.getData());
-                    int y2 = y & 0xF;
+        CompoundTag tile = block.getNbtData();
-                    idArray[y2] = 0;
+        if (tile != null) {
-                }
+            nbtMap.put(index, tile);
-                return true;
-            }
-            case 54:
-            case 130:
-            case 142:
-            case 27:
-            case 137:
-            case 52:
-            case 154:
-            case 84:
-            case 25:
-            case 144:
-            case 138:
-            case 176:
-            case 177:
-            case 63:
-            case 119:
-            case 68:
-            case 323:
-            case 117:
-            case 116:
-            case 28:
-            case 66:
-            case 157:
-            case 61:
-            case 62:
-            case 140:
-            case 146:
-            case 149:
-            case 150:
-            case 158:
-            case 23:
-            case 123:
-            case 124:
-            case 29:
-            case 33:
-            case 151:
-            case 178: {
-                if (block.hasNbtData()) {
-                    nbtMap.put(new IntegerTrio(x, y, z), block.getNbtData());
-                }
-                int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
-                int y2 = y & 0xF;
-                byte[] idArray = ids[i];
-                if (idArray == null) {
-                    idArray = new byte[heights[ylast >> 4]];
-                    ids[i] = idArray;
-                }
-                idArray[y2] = (byte) id;
-                if (FaweCache.hasData(id)) {
-                    int data = block.getData();
-                    if (data == 0) {
-                        return true;
-                    }
-                    if (datas == null) {
-                        datas = new byte[area * ((height + 15) >> 4)][];
-                    }
-                    byte[] dataArray = datas[i];
-                    if (dataArray == null) {
-                        dataArray = datas[i] = new byte[heights[ylast >> 4]];
-                    }
-                    dataArray[y2] = (byte) data;
-                }
-                return true;
-            }
-            case 2:
-            case 4:
-            case 13:
-            case 14:
-            case 15:
-            case 20:
-            case 21:
-            case 22:
-            case 30:
-            case 32:
-            case 37:
-            case 39:
-            case 40:
-            case 41:
-            case 42:
-            case 45:
-            case 46:
-            case 47:
-            case 48:
-            case 49:
-            case 51:
-            case 56:
-            case 57:
-            case 58:
-            case 60:
-            case 7:
-            case 11:
-            case 73:
-            case 74:
-            case 79:
-            case 80:
-            case 81:
-            case 82:
-            case 83:
-            case 85:
-            case 87:
-            case 88:
-            case 101:
-            case 102:
-            case 103:
-            case 110:
-            case 112:
-            case 113:
-            case 121:
-            case 122:
-            case 129:
-            case 133:
-            case 165:
-            case 166:
-            case 169:
-            case 170:
-            case 172:
-            case 173:
-            case 174:
-            case 188:
-            case 189:
-            case 190:
-            case 191:
-            case 192: {
-                int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
-                int y2 = y & 0xF;
-                byte[] idArray = ids[i];
-                if (idArray == null) {
-                    idArray = new byte[heights[ylast >> 4]];
-                    ids[i] = idArray;
-                }
-                idArray[y2] = (byte) id;
-                return true;
-            }
-            default: {
-                int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width));
-                int y2 = y & 0xF;
-                byte[] idArray = ids[i];
-                if (idArray == null) {
-                    idArray = new byte[heights[ylast >> 4]];
-                    ids[i] = idArray;
-                }
-                idArray[y2] = (byte) id;
-                int data = block.getData();
-                if (data == 0) {
-                    return true;
-                }
-                if (datas == null) {
-                    datas = new byte[area * ((height + 15) >> 4)][];
-                }
-                byte[] dataArray = datas[i];
-                if (dataArray == null) {
-                    dataArray = datas[i] = new byte[heights[ylast >> 4]];
-                }
-                dataArray[y2] = (byte) data;
-                return true;
-            }
         }
+        return true;
     }
 
     @Override

core/src/main/java/com/boydti/fawe/object/clipboard/ReadOnlyClipboard.java

@@ -13,19 +13,19 @@ import com.sk89q.worldedit.regions.Region;
 import java.util.Iterator;
 import java.util.List;
 
-public abstract class LazyClipboard extends FaweClipboard {
+public abstract class ReadOnlyClipboard extends FaweClipboard {
     private final Region region;
 
-    public LazyClipboard(Region region) {
+    public ReadOnlyClipboard(Region region) {
         this.region = region;
     }
 
-    public static LazyClipboard of(final EditSession editSession, Region region) {
+    public static ReadOnlyClipboard of(final EditSession editSession, final Region region) {
         final Vector origin = region.getMinimumPoint();
         final int mx = origin.getBlockX();
         final int my = origin.getBlockY();
         final int mz = origin.getBlockZ();
-        return new LazyClipboard(region) {
+        return new ReadOnlyClipboard(region) {
             @Override
             public BaseBlock getBlock(int x, int y, int z) {
                 return editSession.getLazyBlock(mx + x, my + y, mz + z);
@@ -62,6 +62,16 @@ public abstract class LazyClipboard extends FaweClipboard {
         return region;
     }
 
+    @Override
+    public Vector getDimensions() {
+        return region.getMaximumPoint().subtract(region.getMinimumPoint()).add(1, 1, 1);
+    }
+
+    @Override
+    public void setDimensions(Vector dimensions) {
+        throw new UnsupportedOperationException("Clipboard is immutable");
+    }
+
     @Override
     public abstract BaseBlock getBlock(int x, int y, int z);
 
@@ -87,4 +97,19 @@ public abstract class LazyClipboard extends FaweClipboard {
     public boolean remove(ClipboardEntity clipboardEntity) {
         throw new UnsupportedOperationException("Clipboard is immutable");
     }
+
+    @Override
+    public void setId(int index, int id) {
+        throw new UnsupportedOperationException("Clipboard is immutable");
+    }
+
+    @Override
+    public void setData(int index, int data) {
+        throw new UnsupportedOperationException("Clipboard is immutable");
+    }
+
+    @Override
+    public void setAdd(int index, int id) {
+        throw new UnsupportedOperationException("Clipboard is immutable");
+    }
 }

core/src/main/java/com/boydti/fawe/object/schematic/FaweFormat.java

@@ -93,7 +93,7 @@ public class FaweFormat implements ClipboardReader, ClipboardWriter {
                 from = true;
             case 2:
                 small = false;
-            case 1: {
+            case 1: { // Unknown size
                 ox = in.readInt();
                 oz = in.readInt();
                 FaweOutputStream tmp = new FaweOutputStream(new ByteArrayOutputStream(Settings.HISTORY.BUFFER_SIZE));

core/src/main/java/com/boydti/fawe/object/schematic/Schematic.java

@@ -1,6 +1,6 @@
 package com.boydti.fawe.object.schematic;
 
-import com.boydti.fawe.object.clipboard.LazyClipboard;
+import com.boydti.fawe.object.clipboard.ReadOnlyClipboard;
 import com.boydti.fawe.util.EditSessionBuilder;
 import com.sk89q.worldedit.EditSession;
 import com.sk89q.worldedit.MaxChangedBlocksException;
@@ -42,7 +42,7 @@ public class Schematic {
         checkNotNull(region);
         checkNotNull(region.getWorld());
         EditSession session = new EditSessionBuilder(region.getWorld()).allowedRegionsEverywhere().autoQueue(false).build();
-        this.clipboard = new BlockArrayClipboard(region, LazyClipboard.of(session, region));
+        this.clipboard = new BlockArrayClipboard(region, ReadOnlyClipboard.of(session, region));
     }
 
     /**

core/src/main/java/com/boydti/fawe/regions/general/plot/FaweSchematicHandler.java

@@ -3,7 +3,7 @@ package com.boydti.fawe.regions.general.plot;
 import com.boydti.fawe.FaweCache;
 import com.boydti.fawe.object.FaweQueue;
 import com.boydti.fawe.object.RunnableVal2;
-import com.boydti.fawe.object.clipboard.LazyClipboard;
+import com.boydti.fawe.object.clipboard.ReadOnlyClipboard;
 import com.boydti.fawe.util.EditSessionBuilder;
 import com.boydti.fawe.util.SetQueue;
 import com.boydti.fawe.util.TaskManager;
@@ -65,7 +65,7 @@ public class FaweSchematicHandler extends SchematicHandler {
                 final int my = pos1.getY();
                 final int mz = pos1.getZ();
 
-                LazyClipboard clipboard = new LazyClipboard(region) {
+                ReadOnlyClipboard clipboard = new ReadOnlyClipboard(region) {
                     @Override
                     public BaseBlock getBlock(int x, int y, int z) {
                         return editSession.getLazyBlock(mx + x, my + y, mz + z);

core/src/main/java/com/boydti/fawe/util/MainUtil.java

@@ -39,6 +39,7 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -47,9 +48,12 @@ import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
+import net.jpountz.lz4.LZ4Compressor;
 import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
 import net.jpountz.lz4.LZ4InputStream;
 import net.jpountz.lz4.LZ4OutputStream;
+import net.jpountz.lz4.LZ4Utils;
 
 public class MainUtil {
     /*
@@ -102,37 +106,78 @@ public class MainUtil {
     }
 
     public static FaweOutputStream getCompressedOS(OutputStream os, int amount) throws IOException {
+        return getCompressedOS(os, amount, Settings.HISTORY.BUFFER_SIZE);
+    }
+
+    private static final LZ4Factory FACTORY = LZ4Factory.fastestInstance();
+    private static final LZ4Compressor COMPRESSOR = FACTORY.fastCompressor();
+    private static final LZ4FastDecompressor DECOMPRESSOR = FACTORY.fastDecompressor();
+
+    public static int getMaxCompressedLength(int size) {
+        return LZ4Utils.maxCompressedLength(size);
+    }
+
+    public static byte[] compress(byte[] bytes, byte[] buffer, int level) {
+        if (level == 0) {
+            return bytes;
+        }
+        LZ4Compressor compressor = level == 1 ? COMPRESSOR : FACTORY.highCompressor(level);
+        int decompressedLength = bytes.length;
+        if (buffer == null) {
+            int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
+            buffer = new byte[maxCompressedLength];
+        }
+        int compressLen = compressor.compress(bytes, 0, decompressedLength, buffer, 0, buffer.length);
+        return Arrays.copyOf(buffer, compressLen);
+    }
+
+    public static byte[] decompress(byte[] bytes, byte[] buffer, int length, int level) {
+        if (level == 0) {
+            return bytes;
+        }
+        if (buffer == null) {
+            buffer = new byte[length];
+        }
+        DECOMPRESSOR.decompress(bytes, buffer);
+        return buffer;
+    }
+
+    public static FaweOutputStream getCompressedOS(OutputStream os, int amount, int buffer) throws IOException {
         os.write((byte) amount);
-        os = new BufferedOutputStream(os, Settings.HISTORY.BUFFER_SIZE);
+        os = new BufferedOutputStream(os, buffer);
         if (amount == 0) {
             return new FaweOutputStream(os);
         }
         int gzipAmount = amount > 6 ? 1 : 0;
         for (int i = 0; i < gzipAmount; i++) {
-            os = new GZIPOutputStream(os, true);
+            os = new BufferedOutputStream(new GZIPOutputStream(os, buffer, true));
         }
         LZ4Factory factory = LZ4Factory.fastestInstance();
         int fastAmount = 1 + ((amount - 1) % 3);
         for (int i = 0; i < fastAmount; i++) {
-            os = new LZ4OutputStream(os, Settings.HISTORY.BUFFER_SIZE, factory.fastCompressor());
+            os = new LZ4OutputStream(os, buffer, factory.fastCompressor());
         }
         int highAmount = amount > 3 ? 1 : 0;
         for (int i = 0; i < highAmount; i++) {
-            os = new LZ4OutputStream(os, Settings.HISTORY.BUFFER_SIZE, factory.highCompressor());
+            os = new LZ4OutputStream(os, buffer, factory.highCompressor());
         }
         return new FaweOutputStream(os);
     }
 
     public static FaweInputStream getCompressedIS(InputStream is) throws IOException {
+        return getCompressedIS(is, Settings.HISTORY.BUFFER_SIZE);
+    }
+
+    public static FaweInputStream getCompressedIS(InputStream is, int buffer) throws IOException {
         int amount = is.read();
-        is = new BufferedInputStream(is, Settings.HISTORY.BUFFER_SIZE);
+        is = new BufferedInputStream(is, buffer);
         if (amount == 0) {
             return new FaweInputStream(is);
         }
         LZ4Factory factory = LZ4Factory.fastestInstance();
         boolean gzip = amount > 6;
         if (gzip) {
-            is = new GZIPInputStream(is);
+            is = new BufferedInputStream(new GZIPInputStream(is, buffer));
         }
         amount = (1 + ((amount - 1) % 3)) + (amount > 3 ? 1 : 0);
         for (int i = 0; i < amount; i++) {

core/src/main/java/com/sk89q/jnbt/NBTInputStream.java

@@ -19,6 +19,7 @@
 
 package com.sk89q.jnbt;
 
+import com.boydti.fawe.jnbt.NBTStreamer;
 import com.boydti.fawe.object.RunnableVal2;
 import java.io.Closeable;
 import java.io.DataInputStream;
@@ -135,8 +136,15 @@ public final class NBTInputStream implements Closeable {
                     is.skip(length);
                     return;
                 }
-                for (int i = 0; i < length; i++) {
+                if (reader instanceof NBTStreamer.ByteReader) {
-                    reader.run(i, is.readByte());
+                    NBTStreamer.ByteReader byteReader = (NBTStreamer.ByteReader) reader;
+                    for (int i = 0; i < length; i++) {
+                        byteReader.run(i, is.read());
+                    }
+                } else {
+                    for (int i = 0; i < length; i++) {
+                        reader.run(i, is.readByte());
+                    }
                 }
                 return;
             case NBTConstants.TYPE_LIST:

core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java

@@ -22,7 +22,7 @@ package com.sk89q.worldedit.command;
 import com.boydti.fawe.FaweAPI;
 import com.boydti.fawe.config.BBC;
 import com.boydti.fawe.object.RunnableVal2;
-import com.boydti.fawe.object.clipboard.LazyClipboard;
+import com.boydti.fawe.object.clipboard.ReadOnlyClipboard;
 import com.boydti.fawe.util.ImgurUtility;
 import com.sk89q.minecraft.util.commands.Command;
 import com.sk89q.minecraft.util.commands.CommandException;
@@ -111,7 +111,7 @@ public class ClipboardCommands {
         final int mx = origin.getBlockX();
         final int my = origin.getBlockY();
         final int mz = origin.getBlockZ();
-        LazyClipboard lazyClipboard = new LazyClipboard(region) {
+        ReadOnlyClipboard lazyClipboard = new ReadOnlyClipboard(region) {
             @Override
             public BaseBlock getBlock(int x, int y, int z) {
                 return editSession.getLazyBlock(mx + x, my + y, mz + z);

core/src/main/java/com/sk89q/worldedit/command/SchematicCommands.java

@@ -46,8 +46,6 @@ import com.sk89q.worldedit.util.command.parametric.Optional;
 import com.sk89q.worldedit.util.io.file.FilenameException;
 import com.sk89q.worldedit.util.io.file.FilenameResolutionException;
 import com.sk89q.worldedit.world.registry.WorldData;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
@@ -116,7 +114,6 @@ public class SchematicCommands {
                 }
                 in = new FileInputStream(f);
             }
-            in = new BufferedInputStream(in);
             final ClipboardReader reader = format.getReader(in);
             final WorldData worldData = player.getWorld().getWorldData();
             final Clipboard clipboard;
@@ -183,16 +180,14 @@ public class SchematicCommands {
                     target = clipboard;
                 }
 
-                try (BufferedOutputStream bos = new BufferedOutputStream(fos)) {
+                try (ClipboardWriter writer = format.getWriter(fos)) {
-                    try (ClipboardWriter writer = format.getWriter(bos)) {
+                    if (writer instanceof StructureFormat) {
-                        if (writer instanceof StructureFormat) {
+                        ((StructureFormat) writer).write(target, holder.getWorldData(), player.getName());
-                            ((StructureFormat) writer).write(target, holder.getWorldData(), player.getName());
+                    } else {
-                        } else {
+                        writer.write(target, holder.getWorldData());
-                            writer.write(target, holder.getWorldData());
-                        }
-                        log.info(player.getName() + " saved " + f.getCanonicalPath());
-                        BBC.SCHEMATIC_SAVED.send(player, filename);
                     }
+                    log.info(player.getName() + " saved " + f.getCanonicalPath());
+                    BBC.SCHEMATIC_SAVED.send(player, filename);
                 }
             }
         } catch (IllegalArgumentException  e) {

core/src/main/java/com/sk89q/worldedit/extent/clipboard/BlockArrayClipboard.java

@@ -50,20 +50,12 @@ import static com.google.common.base.Preconditions.checkNotNull;
  */
 public class BlockArrayClipboard implements Clipboard {
 
-    private final Region region;
+    private Region region;
-
     public FaweClipboard IMP;
-
+    private Vector size;
-    private final Vector size;
-
-
     private int mx;
     private int my;
     private int mz;
-
-    private int dx;
-    private int dxz;
-
     private Vector origin;
 
     public BlockArrayClipboard(Region region) {
@@ -99,7 +91,19 @@ public class BlockArrayClipboard implements Clipboard {
         checkNotNull(region);
         this.region = region.clone();
         this.size = getDimensions();
+        this.origin = region.getMinimumPoint();
+        this.mx = origin.getBlockX();
+        this.my = origin.getBlockY();
+        this.mz = origin.getBlockZ();
         this.IMP = clipboard;
+    }
+
+    public void init(Region region, FaweClipboard fc) {
+        checkNotNull(region);
+        checkNotNull(fc);
+        this.region = region.clone();
+        this.size = getDimensions();
+        this.IMP = fc;
         this.origin = region.getMinimumPoint();
         this.mx = origin.getBlockX();
         this.my = origin.getBlockY();

core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java

@@ -29,6 +29,8 @@ import com.boydti.fawe.util.MainUtil;
 import com.sk89q.jnbt.NBTConstants;
 import com.sk89q.jnbt.NBTInputStream;
 import com.sk89q.jnbt.NBTOutputStream;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -60,19 +62,21 @@ public enum ClipboardFormat {
     SCHEMATIC("mcedit", "mce", "schematic") {
         @Override
         public ClipboardReader getReader(InputStream inputStream) throws IOException {
-            NBTInputStream nbtStream = new NBTInputStream(new GZIPInputStream(inputStream));
+            inputStream = new BufferedInputStream(inputStream);
+            NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(inputStream)));
             return new SchematicReader(nbtStream);
         }
 
         @Override
         public ClipboardWriter getWriter(OutputStream outputStream) throws IOException {
+            outputStream = new BufferedOutputStream(outputStream);
             GZIPOutputStream gzip;
             if (outputStream instanceof GZIPOutputStream) {
                 gzip = (GZIPOutputStream) outputStream;
             } else {
                 gzip = new GZIPOutputStream(outputStream, true);
             }
-            NBTOutputStream nbtStream = new NBTOutputStream(gzip);
+            NBTOutputStream nbtStream = new NBTOutputStream(new BufferedOutputStream(gzip));
             return new SchematicWriter(nbtStream);
         }
 
@@ -112,19 +116,21 @@ public enum ClipboardFormat {
     STRUCTURE("structure", "nbt") {
         @Override
         public ClipboardReader getReader(InputStream inputStream) throws IOException {
-            NBTInputStream nbtStream = new NBTInputStream(new GZIPInputStream(inputStream));
+            inputStream = new BufferedInputStream(inputStream);
+            NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(inputStream)));
             return new StructureFormat(nbtStream);
         }
 
         @Override
         public ClipboardWriter getWriter(OutputStream outputStream) throws IOException {
+            outputStream = new BufferedOutputStream(outputStream);
             GZIPOutputStream gzip;
             if (outputStream instanceof GZIPOutputStream) {
                 gzip = (GZIPOutputStream) outputStream;
             } else {
                 gzip = new GZIPOutputStream(outputStream, true);
             }
-            NBTOutputStream nbtStream = new NBTOutputStream(gzip);
+            NBTOutputStream nbtStream = new NBTOutputStream(new BufferedOutputStream(gzip));
             return new StructureFormat(nbtStream);
         }
 
@@ -139,6 +145,9 @@ public enum ClipboardFormat {
         }
     },
 
+    /**
+     * Isometric PNG writer
+     */
     PNG("png", "image") {
         @Override
         public ClipboardReader getReader(InputStream inputStream) throws IOException {
@@ -147,7 +156,7 @@ public enum ClipboardFormat {
 
         @Override
         public ClipboardWriter getWriter(OutputStream outputStream) throws IOException {
-            return new PNGWriter(outputStream);
+            return new PNGWriter(new BufferedOutputStream(outputStream));
         }
 
         @Override

core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java

@@ -19,35 +19,15 @@
 
 package com.sk89q.worldedit.extent.clipboard.io;
 
-import com.boydti.fawe.FaweCache;
-import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.jnbt.SchematicStreamer;
-import com.sk89q.jnbt.ByteArrayTag;
 import com.sk89q.jnbt.CompoundTag;
-import com.sk89q.jnbt.IntTag;
-import com.sk89q.jnbt.ListTag;
 import com.sk89q.jnbt.NBTInputStream;
-import com.sk89q.jnbt.NamedTag;
-import com.sk89q.jnbt.ShortTag;
-import com.sk89q.jnbt.StringTag;
 import com.sk89q.jnbt.Tag;
-import com.sk89q.worldedit.BlockVector;
-import com.sk89q.worldedit.Vector;
-import com.sk89q.worldedit.blocks.BaseBlock;
-import com.sk89q.worldedit.entity.BaseEntity;
-import com.sk89q.worldedit.extent.clipboard.BlockArrayClipboard;
 import com.sk89q.worldedit.extent.clipboard.Clipboard;
-import com.sk89q.worldedit.regions.CuboidRegion;
-import com.sk89q.worldedit.regions.Region;
-import com.sk89q.worldedit.util.Location;
 import com.sk89q.worldedit.world.registry.WorldData;
-import com.sk89q.worldedit.world.storage.NBTConversions;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.annotation.Nullable;
 
@@ -78,191 +58,7 @@ public class SchematicReader implements ClipboardReader {
     }
 
     public Clipboard read(WorldData data, UUID clipboardId) throws IOException {
-        if (Settings.CLIPBOARD.USE_DISK) {
+        return new SchematicStreamer(inputStream, clipboardId).getClipboard();
-            return new SchematicStreamer(inputStream, clipboardId).getClipboard();
-        }
-        // Schematic tag
-        NamedTag rootTag = inputStream.readNamedTag();
-        if (!rootTag.getName().equals("Schematic")) {
-            throw new IOException("Tag 'Schematic' does not exist or is not first");
-        }
-        CompoundTag schematicTag = (CompoundTag) rootTag.getTag();
-
-        // Check
-        Map<String, Tag> schematic = schematicTag.getValue();
-        if (!schematic.containsKey("Blocks")) {
-            throw new IOException("Schematic file is missing a 'Blocks' tag");
-        }
-
-        // Check type of Schematic
-        String materials = requireTag(schematic, "Materials", StringTag.class).getValue();
-        if (!materials.equals("Alpha")) {
-            throw new IOException("Schematic file is not an Alpha schematic");
-        }
-
-        // ====================================================================
-        // Metadata
-        // ====================================================================
-
-        Vector origin;
-        Region region;
-
-        // Get information
-        short width = requireTag(schematic, "Width", ShortTag.class).getValue();
-        short height = requireTag(schematic, "Height", ShortTag.class).getValue();
-        short length = requireTag(schematic, "Length", ShortTag.class).getValue();
-
-        try {
-            int originX = requireTag(schematic, "WEOriginX", IntTag.class).getValue();
-            int originY = requireTag(schematic, "WEOriginY", IntTag.class).getValue();
-            int originZ = requireTag(schematic, "WEOriginZ", IntTag.class).getValue();
-            Vector min = new Vector(originX, originY, originZ);
-
-            int offsetX = requireTag(schematic, "WEOffsetX", IntTag.class).getValue();
-            int offsetY = requireTag(schematic, "WEOffsetY", IntTag.class).getValue();
-            int offsetZ = requireTag(schematic, "WEOffsetZ", IntTag.class).getValue();
-            Vector offset = new Vector(offsetX, offsetY, offsetZ);
-
-            origin = min.subtract(offset);
-            region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE));
-        } catch (IOException ignored) {
-            origin = new Vector(0, 0, 0);
-            region = new CuboidRegion(origin, origin.add(width, height, length).subtract(Vector.ONE));
-        }
-
-        // ====================================================================
-        // Blocks
-        // ====================================================================
-
-        // Get blocks
-        byte[] blockId = requireTag(schematic, "Blocks", ByteArrayTag.class).getValue();
-        byte[] blockData = requireTag(schematic, "Data", ByteArrayTag.class).getValue();
-        byte[] addId = null;
-
-        // We support 4096 block IDs using the same method as vanilla Minecraft, where
-        // the highest 4 bits are stored in a separate byte array.
-        if (schematic.containsKey("AddBlocks")) {
-            addId = requireTag(schematic, "AddBlocks", ByteArrayTag.class).getValue();
-        }
-
-        // Need to pull out tile entities
-        List<Tag> tileEntities = requireTag(schematic, "TileEntities", ListTag.class).getValue();
-        Map<BlockVector, Map<String, Tag>> tileEntitiesMap = new HashMap<BlockVector, Map<String, Tag>>();
-
-        for (Tag tag : tileEntities) {
-            if (!(tag instanceof CompoundTag)) continue;
-            CompoundTag t = (CompoundTag) tag;
-
-            int x = 0;
-            int y = 0;
-            int z = 0;
-
-            Map<String, Tag> values = new HashMap<String, Tag>();
-
-            for (Map.Entry<String, Tag> entry : t.getValue().entrySet()) {
-                if (entry.getKey().equals("x")) {
-                    if (entry.getValue() instanceof IntTag) {
-                        x = ((IntTag) entry.getValue()).getValue();
-                    }
-                } else if (entry.getKey().equals("y")) {
-                    if (entry.getValue() instanceof IntTag) {
-                        y = ((IntTag) entry.getValue()).getValue();
-                    }
-                } else if (entry.getKey().equals("z")) {
-                    if (entry.getValue() instanceof IntTag) {
-                        z = ((IntTag) entry.getValue()).getValue();
-                    }
-                }
-
-                values.put(entry.getKey(), entry.getValue());
-            }
-
-            BlockVector vec = new BlockVector(x, y, z);
-            tileEntitiesMap.put(vec, values);
-        }
-
-        BlockArrayClipboard clipboard = new BlockArrayClipboard(region, clipboardId);
-        clipboard.setOrigin(origin);
-
-        // Don't log a torrent of errors
-        int failedBlockSets = 0;
-
-        Vector min = region.getMinimumPoint();
-        int mx = min.getBlockX();
-        int my = min.getBlockY();
-        int mz = min.getBlockZ();
-
-        BlockVector pt = new BlockVector(0, 0, 0);
-
-        int i;
-        for (int y = 0; y < height; y++) {
-            int yy = y + my;
-            final int i1 = y * width * length;
-            for (int z = 0; z < length; z++) {
-                int zz = z + mz;
-                final int i2 = (z * width) + i1;
-                for (int x = 0; x < width; x++) {
-                    i = i2 + x;
-                    int xx = x + mx;
-                    int id = blockId[i] & 0xFF;
-                    int db = blockData[i];
-                    if (addId != null) {
-                        if ((i & 1) == 0) {
-                            id += ((addId[i >> 1] & 0x0F) << 8);
-                        } else {
-                            id += ((addId[i >> 1] & 0xF0) << 4);
-                        }
-                    }
-                    BaseBlock block = FaweCache.getBlock(id, db);
-                    if (FaweCache.hasNBT(id)) {
-                        pt.x = x;
-                        pt.y = y;
-                        pt.z = z;
-                        if (tileEntitiesMap.containsKey(pt)) {
-                            block = new BaseBlock(block.getId(), block.getData());
-                            block.setNbtData(new CompoundTag(tileEntitiesMap.get(pt)));
-                        }
-                    }
-                    try {
-                        clipboard.setBlock(xx, yy, zz, block);
-                    } catch (Exception e) {
-                        switch (failedBlockSets) {
-                            case 0:
-                                log.log(Level.WARNING, "Failed to set block on a Clipboard", e);
-                                break;
-                            case 1:
-                                log.log(Level.WARNING, "Failed to set block on a Clipboard (again) -- no more messages will be logged", e);
-                                break;
-                            default:
-                        }
-                        failedBlockSets++;
-                    }
-                }
-            }
-        }
-
-        // ====================================================================
-        // Entities
-        // ====================================================================
-
-        try {
-            List<Tag> entityTags = requireTag(schematic, "Entities", ListTag.class).getValue();
-
-            for (Tag tag : entityTags) {
-                if (tag instanceof CompoundTag) {
-                    CompoundTag compound = (CompoundTag) tag;
-                    String id = compound.getString("id");
-                    Location location = NBTConversions.toLocation(clipboard, compound.getListTag("Pos"), compound.getListTag("Rotation"));
-                    if (!id.isEmpty()) {
-                        BaseEntity state = new BaseEntity(id, compound);
-                        clipboard.createEntity(location, state);
-                    }
-                }
-            }
-        } catch (IOException ignored) { // No entities? No problem
-        }
-
-        return clipboard;
     }
 
     private static <T extends Tag> T requireTag(Map<String, Tag> items, String key, Class<T> expected) throws IOException {

core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicWriter.java

@@ -145,7 +145,7 @@ public class SchematicWriter implements ClipboardWriter {
         schematic.put("WEOffsetZ", new IntTag(offset.getBlockZ()));
 
         final byte[] blocks = new byte[width * height * length];
-        byte[] addBlocks = null;
+        byte[] addBlocks;
         final byte[] blockData = new byte[width * height * length];
         final List<Tag> tileEntities = new ArrayList<Tag>();
         // Precalculate index vars

core/src/main/java/net/jpountz/lz4/LZ4Utils.java

@@ -19,12 +19,12 @@ import static net.jpountz.lz4.LZ4Constants.HASH_LOG_64K;
 import static net.jpountz.lz4.LZ4Constants.HASH_LOG_HC;
 import static net.jpountz.lz4.LZ4Constants.MIN_MATCH;
 
-enum LZ4Utils {
+public enum LZ4Utils {
   ;
 
   private static final int MAX_INPUT_SIZE = 0x7E000000;
 
-  static int maxCompressedLength(int length) {
+  public static int maxCompressedLength(int length) {
     if (length < 0) {
       throw new IllegalArgumentException("length must be >= 0, got " + length);
     } else if (length >= MAX_INPUT_SIZE) {
@@ -33,19 +33,19 @@ enum LZ4Utils {
     return length + length / 255 + 16;
   }
 
-  static int hash(int i) {
+  public static int hash(int i) {
     return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG);
   }
 
-  static int hash64k(int i) {
+  public static int hash64k(int i) {
     return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_64K);
   }
 
-  static int hashHC(int i) {
+  public static int hashHC(int i) {
     return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_HC);
   }
 
-  static class Match {
+  public static class Match {
     int start, ref, len;
 
     void fix(int correction) {
@@ -59,7 +59,7 @@ enum LZ4Utils {
     }
   }
 
-  static void copyTo(Match m1, Match m2) {
+  public static void copyTo(Match m1, Match m2) {
     m2.len = m1.len;
     m2.start = m1.start;
     m2.ref = m1.ref;