From c98d07039d09479aec9278c932817f551b7d5364 Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Sat, 6 Aug 2016 16:42:40 +1000 Subject: [PATCH] Buffered random access clipboard compression + schematic -> clipboard streaming Also adds CPUOptimizedClipboard which has no extra compression Note: Performance degrades if access is actually random (the buffering optimizes sequential r/w) Removing stream in favor of clipboard compression / disk - In order to stream a schematic, it would need to be fully read 3 times as tags are not ordered (dimensions -> block ids -> data + tiles + entities) - Much faster just using disk / memory as an intermediate step --- bukkit110/src/main/resources/plugin.yml | 3 - bukkit1710/src/main/resources/plugin.yml | 3 - bukkit18/src/main/resources/plugin.yml | 3 - bukkit19/src/main/resources/plugin.yml | 3 - core/src/main/java/com/boydti/fawe/Fawe.java | 3 +- .../java/com/boydti/fawe/command/Stream.java | 37 -- .../java/com/boydti/fawe/config/Settings.java | 11 +- .../com/boydti/fawe/jnbt/NBTStreamer.java | 9 + .../boydti/fawe/jnbt/SchematicStreamer.java | 214 ++++--- .../clipboard/CPUOptimizedClipboard.java | 177 ++++++ .../clipboard/DiskOptimizedClipboard.java | 38 +- .../fawe/object/clipboard/FaweClipboard.java | 10 + .../clipboard/MemoryOptimizedClipboard.java | 529 +++++++++--------- ...yClipboard.java => ReadOnlyClipboard.java} | 33 +- .../fawe/object/schematic/FaweFormat.java | 2 +- .../fawe/object/schematic/Schematic.java | 4 +- .../general/plot/FaweSchematicHandler.java | 4 +- .../java/com/boydti/fawe/util/MainUtil.java | 57 +- .../java/com/sk89q/jnbt/NBTInputStream.java | 12 +- .../worldedit/command/ClipboardCommands.java | 4 +- .../worldedit/command/SchematicCommands.java | 19 +- .../extent/clipboard/BlockArrayClipboard.java | 24 +- .../extent/clipboard/io/ClipboardFormat.java | 19 +- .../extent/clipboard/io/SchematicReader.java | 206 +------ .../extent/clipboard/io/SchematicWriter.java | 2 +- .../main/java/net/jpountz/lz4/LZ4Utils.java | 14 +- 26 files changed, 758 insertions(+), 682 deletions(-) delete mode 100644 core/src/main/java/com/boydti/fawe/command/Stream.java create mode 100644 core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java rename core/src/main/java/com/boydti/fawe/object/clipboard/{LazyClipboard.java => ReadOnlyClipboard.java} (74%) diff --git a/bukkit110/src/main/resources/plugin.yml b/bukkit110/src/main/resources/plugin.yml index 2a71074a..6c351f9f 100644 --- a/bukkit110/src/main/resources/plugin.yml +++ b/bukkit110/src/main/resources/plugin.yml @@ -12,9 +12,6 @@ commands: description: (FAWE) Bypass WorldEdit processing and area restrictions aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere] usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass" - stream: - description: (FAWE) Stream a schematic into the world - aliases: [/stream] fawe: description: (FAWE) Reload the plugin aliases: [/fawe,/fawereload] diff --git a/bukkit1710/src/main/resources/plugin.yml b/bukkit1710/src/main/resources/plugin.yml index 144b80f2..362758b2 100644 --- a/bukkit1710/src/main/resources/plugin.yml +++ b/bukkit1710/src/main/resources/plugin.yml @@ -12,9 +12,6 @@ commands: description: (FAWE) Bypass WorldEdit processing and area restrictions aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere] usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass" - stream: - description: (FAWE) Stream a schematic into the world - aliases: [/stream] fawe: description: (FAWE) Reload the plugin aliases: [/fawe,/fawereload] diff --git a/bukkit18/src/main/resources/plugin.yml b/bukkit18/src/main/resources/plugin.yml index 6f93d1b0..491e5c3d 100644 --- a/bukkit18/src/main/resources/plugin.yml +++ b/bukkit18/src/main/resources/plugin.yml @@ -12,9 +12,6 @@ commands: description: (FAWE) Bypass WorldEdit processing and area restrictions aliases: [weanywhere,worldeditanywhere,/wea,/weanywhere,/worldeditanywhere] usage: "Vault is required for the toggle. Optionally, you can set the permission fawe.bypass" - stream: - description: (FAWE) Stream a schematic into the world - aliases: [/stream] fawe: description: (FAWE) Reload the plugin aliases: [/fawe,/fawereload] diff --git a/bukkit19/src/main/resources/plugin.yml b/bukkit19/src/main/resources/plugin.yml index 7c172f18..d8b87c7d 100644 --- a/bukkit19/src/main/resources/plugin.yml +++ b/bukkit19/src/main/resources/plugin.yml @@ -15,9 +15,6 @@ commands: fixlighting: description: (FAWE) Fix the lighting in your current chunk aliases: [/fixlighting] - stream: - description: (FAWE) Stream a schematic into the world - aliases: [/stream] fawe: description: (FAWE) Reload the plugin aliases: [/fawe,/fawereload] diff --git a/core/src/main/java/com/boydti/fawe/Fawe.java b/core/src/main/java/com/boydti/fawe/Fawe.java index c4622e03..54056c8d 100644 --- a/core/src/main/java/com/boydti/fawe/Fawe.java +++ b/core/src/main/java/com/boydti/fawe/Fawe.java @@ -2,7 +2,6 @@ package com.boydti.fawe; import com.boydti.fawe.command.Cancel; import com.boydti.fawe.command.Reload; -import com.boydti.fawe.command.Stream; import com.boydti.fawe.command.Wea; import com.boydti.fawe.command.WorldEditRegion; import com.boydti.fawe.config.BBC; @@ -68,6 +67,7 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import javax.management.InstanceAlreadyExistsException; import javax.management.Notification; import javax.management.NotificationEmitter; @@ -226,7 +226,6 @@ public class Fawe { private void setupCommands() { this.IMP.setupCommand("wea", new Wea()); - this.IMP.setupCommand("stream", new Stream()); this.IMP.setupCommand("select", new WorldEditRegion()); this.IMP.setupCommand("fawe", new Reload()); this.IMP.setupCommand("fcancel", new Cancel()); diff --git a/core/src/main/java/com/boydti/fawe/command/Stream.java b/core/src/main/java/com/boydti/fawe/command/Stream.java deleted file mode 100644 index 5412a9a0..00000000 --- a/core/src/main/java/com/boydti/fawe/command/Stream.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.boydti.fawe.command; - -import com.boydti.fawe.Fawe; -import com.boydti.fawe.FaweAPI; -import com.boydti.fawe.config.BBC; -import com.boydti.fawe.object.FaweCommand; -import com.boydti.fawe.object.FawePlayer; -import java.io.File; - -public class Stream extends FaweCommand { - - public Stream() { - super("fawe.stream"); - } - - @Override - public boolean execute(final FawePlayer player, final String... args) { - if (player == null) { - return false; - } - if (args.length != 1) { - BBC.COMMAND_SYNTAX.send(player, "/stream "); - return false; - } - if (!args[0].endsWith(".schematic")) { - args[0] += ".schematic"; - } - final File file = Fawe.get().getWorldEdit().getWorkingDirectoryFile(Fawe.get().getWorldEdit().getConfiguration().saveDir + File.separator + args[0]); - if (!file.exists()) { - BBC.SCHEMATIC_NOT_FOUND.send(player, args[0]); - return false; - } - FaweAPI.streamSchematic(file, player.getLocation()); - BBC.SCHEMATIC_PASTING.send(player); - return true; - } -} diff --git a/core/src/main/java/com/boydti/fawe/config/Settings.java b/core/src/main/java/com/boydti/fawe/config/Settings.java index 4b8506c8..c668361d 100644 --- a/core/src/main/java/com/boydti/fawe/config/Settings.java +++ b/core/src/main/java/com/boydti/fawe/config/Settings.java @@ -201,8 +201,17 @@ public class Settings extends Config { } public static class CLIPBOARD { + @Comment("Store the clipboard on disk instead of memory") public static boolean USE_DISK = false; - + @Comment({ + "Compress the clipboard to reduce the size:", + " - TODO: Buffered random access with compression is not implemented on disk yet", + " - 0 = No compression", + " - 1 = Fast compression", + " - 2-17 = Slower compression" + }) + public static int COMPRESSION_LEVEL = 1; + @Comment("Number of days to keep history on disk before deleting it") public static int DELETE_AFTER_DAYS = 1; } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java b/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java index 59a4c043..d1906e87 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/NBTStreamer.java @@ -46,4 +46,13 @@ public class NBTStreamer { return node; } } + + public static abstract class ByteReader extends RunnableVal2 { + @Override + public void run(Integer index, Integer value) { + run(index, value); + } + + public abstract void run(int index, int byteValue); + } } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/SchematicStreamer.java b/core/src/main/java/com/boydti/fawe/jnbt/SchematicStreamer.java index 37cce0f8..38419b71 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/SchematicStreamer.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/SchematicStreamer.java @@ -1,7 +1,11 @@ package com.boydti.fawe.jnbt; +import com.boydti.fawe.config.Settings; import com.boydti.fawe.object.RunnableVal2; +import com.boydti.fawe.object.clipboard.CPUOptimizedClipboard; import com.boydti.fawe.object.clipboard.DiskOptimizedClipboard; +import com.boydti.fawe.object.clipboard.FaweClipboard; +import com.boydti.fawe.object.clipboard.MemoryOptimizedClipboard; import com.sk89q.jnbt.CompoundTag; import com.sk89q.jnbt.ListTag; import com.sk89q.jnbt.NBTInputStream; @@ -12,76 +16,19 @@ import com.sk89q.worldedit.extent.clipboard.Clipboard; import com.sk89q.worldedit.regions.CuboidRegion; import java.io.IOException; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; public class SchematicStreamer extends NBTStreamer { private final UUID uuid; - public SchematicStreamer(NBTInputStream stream, UUID uuid) throws IOException { + public SchematicStreamer(NBTInputStream stream, UUID uuid) { super(stream); this.uuid = uuid; - addReader("Schematic.Height", new RunnableVal2() { - @Override - public void run(Integer index, Short value) { - height = (value); - } - }); - addReader("Schematic.Width", new RunnableVal2() { - @Override - public void run(Integer index, Short value) { - width = (value); - } - }); - addReader("Schematic.Length", new RunnableVal2() { - @Override - public void run(Integer index, Short value) { - length = (value); - } - }); - final AtomicInteger originX = new AtomicInteger(); - final AtomicInteger originY = new AtomicInteger(); - final AtomicInteger originZ = new AtomicInteger(); - addReader("Schematic.WEOriginX", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - originX.set(value); - } - }); - addReader("Schematic.WEOriginY", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - originY.set(value); - } - }); - addReader("Schematic.WEOriginZ", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - originZ.set(value); - } - }); - final AtomicInteger offsetX = new AtomicInteger(); - final AtomicInteger offsetY = new AtomicInteger(); - final AtomicInteger offsetZ = new AtomicInteger(); - addReader("Schematic.WEOffsetX", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - offsetX.set(value); - } - }); - addReader("Schematic.WEOffsetY", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - offsetY.set(value); - } - }); - addReader("Schematic.WEOffsetZ", new RunnableVal2() { - @Override - public void run(Integer index, Integer value) { - offsetZ.set(value); - } - }); - // Blocks - RunnableVal2 initializer = new RunnableVal2() { + clipboard = new BlockArrayClipboard(new CuboidRegion(new Vector(0, 0, 0), new Vector(0, 0, 0)), fc); + } + + public void addBlockReaders() { + final long start = System.currentTimeMillis(); + NBTStreamReader initializer = new NBTStreamReader() { @Override public void run(Integer length, Integer type) { setupClipboard(length); @@ -90,27 +37,25 @@ public class SchematicStreamer extends NBTStreamer { addReader("Schematic.Blocks.?", initializer); addReader("Schematic.Data.?", initializer); addReader("Schematic.AddBlocks.?", initializer); - addReader("Schematic.Blocks.#", new RunnableVal2() { - int i; + addReader("Schematic.Blocks.#", new ByteReader() { @Override - public void run(Integer index, Byte value) { - fc.setId(i++, value); + public void run(int index, int value) { + fc.setId(index, value); } }); - addReader("Schematic.Data.#", new RunnableVal2() { - int i; + addReader("Schematic.Data.#", new ByteReader() { @Override - public void run(Integer index, Byte value) { - fc.setData(i++, value); + public void run(int index, int value) { + fc.setData(index, value); } }); - addReader("Schematic.AddBlocks.#", new RunnableVal2() { - int i; + addReader("Schematic.AddBlocks.#", new ByteReader() { @Override - public void run(Integer index, Byte value) { - fc.setAdd(i++, value); + public void run(int index, int value) { + fc.setAdd(index, value); } }); + // Tiles addReader("Schematic.TileEntities.#", new RunnableVal2() { @Override @@ -141,35 +86,124 @@ public class SchematicStreamer extends NBTStreamer { fc.createEntity(null, positionTag.asDouble(0), positionTag.asDouble(1), positionTag.asDouble(2), (float) directionTag.asDouble(0), (float) directionTag.asDouble(1), state); } }); - readFully(); - Vector min = new Vector(originX.get(), originY.get(), originZ.get()); - Vector offset = new Vector(offsetX.get(), offsetY.get(), offsetZ.get()); - Vector origin = min.subtract(offset); - Vector dimensions = new Vector(width, height, length); - fc.setDimensions(dimensions); - CuboidRegion region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE)); - clipboard = new BlockArrayClipboard(region, fc); - clipboard.setOrigin(origin); + } + + public void addDimensionReaders() { + addReader("Schematic.Height", new RunnableVal2() { + @Override + public void run(Integer index, Short value) { + height = (value); + } + }); + addReader("Schematic.Width", new RunnableVal2() { + @Override + public void run(Integer index, Short value) { + width = (value); + } + }); + addReader("Schematic.Length", new RunnableVal2() { + @Override + public void run(Integer index, Short value) { + length = (value); + } + }); + addReader("Schematic.WEOriginX", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + originX = (value); + } + }); + addReader("Schematic.WEOriginY", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + originY = (value); + } + }); + addReader("Schematic.WEOriginZ", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + originZ = (value); + } + }); + addReader("Schematic.WEOffsetX", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + offsetX = (value); + } + }); + addReader("Schematic.WEOffsetY", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + offsetY = (value); + } + }); + addReader("Schematic.WEOffsetZ", new RunnableVal2() { + @Override + public void run(Integer index, Integer value) { + offsetZ = (value); + } + }); } private int height; private int width; private int length; - private Clipboard clipboard; - private DiskOptimizedClipboard fc; + private int originX; + private int originY; + private int originZ; - private DiskOptimizedClipboard setupClipboard(int size) { + private int offsetX; + private int offsetY; + private int offsetZ; + + private BlockArrayClipboard clipboard; + private FaweClipboard fc; + + private FaweClipboard setupClipboard(int size) { if (fc != null) { if (fc.getDimensions().getX() == 0) { fc.setDimensions(new Vector(size, 1, 1)); } return fc; } - return fc = new DiskOptimizedClipboard(size, 1, 1, uuid); + if (Settings.CLIPBOARD.USE_DISK) { + return fc = new DiskOptimizedClipboard(size, 1, 1, uuid); + } else if (Settings.CLIPBOARD.COMPRESSION_LEVEL == 0) { + return fc = new CPUOptimizedClipboard(size, 1, 1); + } else { + return fc = new MemoryOptimizedClipboard(size, 1, 1); + } } - public Clipboard getClipboard() { + public Vector getOrigin() { + return new Vector(originX, originY, originZ); + } + + public Vector getOffset() { + return new Vector(offsetX, offsetY, offsetZ); + } + + public Vector getDimensions() { + return new Vector(width, height, length); + } + + public void setClipboard(FaweClipboard clipboard) { + this.fc = clipboard; + } + + public Clipboard getClipboard() throws IOException { + addDimensionReaders(); + addBlockReaders(); + readFully(); + Vector min = new Vector(originX, originY, originZ); + Vector offset = new Vector(offsetX, offsetY, offsetZ); + Vector origin = min.subtract(offset); + Vector dimensions = new Vector(width, height, length); + fc.setDimensions(dimensions); + CuboidRegion region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE)); + clipboard.init(region, fc); + clipboard.setOrigin(origin); return clipboard; } } diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java new file mode 100644 index 00000000..ca5d6517 --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java @@ -0,0 +1,177 @@ +package com.boydti.fawe.object.clipboard; + +import com.boydti.fawe.FaweCache; +import com.boydti.fawe.object.RunnableVal2; +import com.sk89q.jnbt.CompoundTag; +import com.sk89q.worldedit.Vector; +import com.sk89q.worldedit.blocks.BaseBlock; +import com.sk89q.worldedit.entity.BaseEntity; +import com.sk89q.worldedit.entity.Entity; +import com.sk89q.worldedit.extent.Extent; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class CPUOptimizedClipboard extends FaweClipboard { + public CPUOptimizedClipboard(int width, int height, int length) { + this.width = width; + this.height = height; + this.length = length; + this.area = width * length; + this.volume = area * height; + ids = new byte[volume]; + datas = new byte[volume]; + nbtMap = new HashMap<>(); + entities = new HashSet<>(); + } + + private int length; + private int height; + private int width; + private int area; + private int volume; + private byte[] ids; + private byte[] datas; + private byte[] add; + private final HashMap nbtMap; + private final HashSet entities; + + public int getId(int index) { + if (add != null) { + return ids[index] & 0xFF + add[index] & 0xFF; + } + return ids[index] & 0xFF; + } + + public int getData(int index) { + return datas[index]; + } + + @Override + public void setDimensions(Vector dimensions) { + width = dimensions.getBlockX(); + height = dimensions.getBlockY(); + length = dimensions.getBlockZ(); + area = width * length; + } + + @Override + public Vector getDimensions() { + return new Vector(width, height, length); + } + + @Override + public void setAdd(int index, int value) { + if (value == 0) { + return; + } + if (this.add == null) { + add = new byte[volume]; + } + add[index] = (byte) value; + } + + @Override + public void setId(int index, int value) { + ids[index] = (byte) value; + } + + @Override + public void setData(int index, int value) { + datas[index] = (byte) value; + } + + private int ylast; + private int ylasti; + private int zlast; + private int zlasti; + + public int getIndex(int x, int y, int z) { + return x + ((ylast == y) ? ylasti : (ylasti = (ylast = y) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); + } + + @Override + public BaseBlock getBlock(int x, int y, int z) { + int index = getIndex(x, y, z); + return getBlock(index); + } + + public BaseBlock getBlock(int index) { + int id = getId(index); + if (id == 0) { + return FaweCache.CACHE_BLOCK[0]; + } + BaseBlock block; + if (FaweCache.hasData(id)) { + block = FaweCache.getBlock(id, getData(index)); + } else { + block = FaweCache.getBlock(id, 0); + } + if (FaweCache.hasNBT(id)) { + CompoundTag nbt = nbtMap.get(index); + if (nbt != null) { + block = new BaseBlock(block.getId(), block.getData()); + block.setNbtData(nbt); + } + } + return block; + } + + @Override + public void forEach(final RunnableVal2 task, boolean air) { + task.value1 = new Vector(0, 0, 0); + for (int y = 0, index = 0; y < height; y++) { + for (int z = 0; z < length; z++) { + for (int x = 0; x < width; x++, index++) { + task.value2 = getBlock(index); + if (!air && task.value2.getId() == 0) { + continue; + } + task.value1.x = x; + task.value1.y = y; + task.value1.z = z; + task.run(); + } + } + } + } + + @Override + public boolean setTile(int x, int y, int z, CompoundTag tag) { + nbtMap.put(getIndex(x, y, z), tag); + return true; + } + + @Override + public boolean setBlock(int x, int y, int z, BaseBlock block) { + return setBlock(getIndex(x, y, z), block); + } + + public boolean setBlock(int index, BaseBlock block) { + setId(index, (byte) block.getId()); + setData(index, (byte) block.getData()); + CompoundTag tile = block.getNbtData(); + if (tile != null) { + nbtMap.put(index, tile); + } + return true; + } + + @Override + public Entity createEntity(Extent world, double x, double y, double z, float yaw, float pitch, BaseEntity entity) { + FaweClipboard.ClipboardEntity ret = new ClipboardEntity(world, x, y, z, yaw, pitch, entity); + entities.add(ret); + return ret; + } + + @Override + public List getEntities() { + return new ArrayList<>(entities); + } + + @Override + public boolean remove(ClipboardEntity clipboardEntity) { + return entities.remove(clipboardEntity); + } +} diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java index eb73762c..e10ff24a 100644 --- a/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java @@ -76,6 +76,7 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { autoCloseTask(); } + @Override public Vector getDimensions() { return new Vector(width, height, length); } @@ -141,6 +142,7 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { } } + @Override public void setDimensions(Vector dimensions) { try { if (raf == null) { @@ -331,7 +333,8 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { return false; } - public boolean setId(int i, int id) { + @Override + public void setId(int i, int id) { try { if (raf == null) { open(); @@ -344,14 +347,12 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { int combined = FaweCache.getData(raf.readChar()) + (id << 4); raf.seekUnsafe(raf.getFilePointer() - 2); raf.writeChar(combined); - return true; } catch (Exception e) { MainUtil.handleError(e); } - return false; } - public boolean setCombined(int i, int combined) { + public void setCombined(int i, int combined) { try { if (raf == null) { open(); @@ -362,14 +363,13 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { } last = i; raf.writeChar(combined); - return true; } catch (Exception e) { MainUtil.handleError(e); } - return false; } - public boolean setAdd(int i, int add) { + @Override + public void setAdd(int i, int add) { try { if (raf == null) { open(); @@ -382,31 +382,13 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { int combined = raf.readChar() + (add << 4); raf.seekUnsafe(raf.getFilePointer() - 2); raf.writeChar(combined); - return true; } catch (Exception e) { MainUtil.handleError(e); } - return false; } - public int getCombined(int i) { - try { - if (raf == null) { - open(); - } - if (i != last + 1) { - raf.seek((HEADER_SIZE) + (i << 1)); - lastAccessed = System.currentTimeMillis(); - } - last = i; - return raf.readChar(); - } catch (Exception e) { - MainUtil.handleError(e); - } - return 0; - } - - public boolean setData(int i, int data) { + @Override + public void setData(int i, int data) { try { if (raf == null) { open(); @@ -419,11 +401,9 @@ public class DiskOptimizedClipboard extends FaweClipboard implements Closeable { int combined = (FaweCache.getId(raf.readChar()) << 4) + data; raf.seekUnsafe(raf.getFilePointer() - 2); raf.writeChar(combined); - return true; } catch (Exception e) { MainUtil.handleError(e); } - return false; } @Override diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/FaweClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/FaweClipboard.java index 6fea42af..89097e88 100644 --- a/core/src/main/java/com/boydti/fawe/object/clipboard/FaweClipboard.java +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/FaweClipboard.java @@ -19,6 +19,12 @@ public abstract class FaweClipboard { public abstract boolean setBlock(int x, int y, int z, BaseBlock block); + public abstract void setId(int index, int id); + + public abstract void setData(int index, int data); + + public abstract void setAdd(int index, int id); + public abstract boolean setTile(int x, int y, int z, CompoundTag tag); public abstract Entity createEntity(Extent world, double x, double y, double z, float yaw, float pitch, BaseEntity entity); @@ -29,6 +35,10 @@ public abstract class FaweClipboard { public void setOrigin(Vector offset) {} // Do nothing + public abstract void setDimensions(Vector dimensions); + + public abstract Vector getDimensions(); + /** * The locations provided are relative to the clipboard min * @param task diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java index f989ea77..1a18ad6b 100644 --- a/core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java @@ -1,11 +1,10 @@ package com.boydti.fawe.object.clipboard; import com.boydti.fawe.FaweCache; -import com.boydti.fawe.object.IntegerTrio; +import com.boydti.fawe.config.Settings; import com.boydti.fawe.object.RunnableVal2; +import com.boydti.fawe.util.MainUtil; import com.sk89q.jnbt.CompoundTag; -import com.sk89q.worldedit.BlockVector; -import com.sk89q.worldedit.EditSession; import com.sk89q.worldedit.Vector; import com.sk89q.worldedit.blocks.BaseBlock; import com.sk89q.worldedit.entity.BaseEntity; @@ -17,30 +16,239 @@ import java.util.HashSet; import java.util.List; public class MemoryOptimizedClipboard extends FaweClipboard { - protected int length; - protected int height; - protected int width; - protected int area; - // x,z,y+15>>4 | y&15 - private final byte[][] ids; - private final byte[] heights; + public static final int BLOCK_SIZE = 1048576; + public static final int BLOCK_MASK = 1048575; + public static final int BLOCK_SHIFT = 20; + + private int length; + private int height; + private int width; + private int area; + private int volume; + + private byte[][] ids; private byte[][] datas; - private final HashMap nbtMap; + private byte[][] add; + + private byte[] buffer = new byte[MainUtil.getMaxCompressedLength(BLOCK_SIZE)]; + + private final HashMap nbtMap; private final HashSet entities; + private int lastIdsI = -1; + private int lastDatasI = -1; + private int lastAddI = -1; + + private byte[] lastIds; + private byte[] lastDatas; + private byte[] lastAdd; + + private boolean saveIds = false; + private boolean saveDatas = false; + private boolean saveAdd = false; + + private int compressionLevel; + public MemoryOptimizedClipboard(int width, int height, int length) { + this(width, height, length, Settings.CLIPBOARD.COMPRESSION_LEVEL); + } + + public MemoryOptimizedClipboard(int width, int height, int length, int compressionLevel) { this.width = width; this.height = height; this.length = length; this.area = width * length; - heights = new byte[(height + 15) >> 4]; - for (int y = 0; y < ((height + 15) >> 4); y++) { - heights[y] = (byte) Math.min(16, height - (y << 4)); - } - ids = new byte[width * length * ((height + 15) >> 4)][]; + this.volume = area * height; + ids = new byte[1 + (volume >> BLOCK_SHIFT)][]; + datas = new byte[1 + (volume >> BLOCK_SHIFT)][]; nbtMap = new HashMap<>(); entities = new HashSet<>(); + this.compressionLevel = compressionLevel; + } + + public int getId(int index) { + int i = index >> BLOCK_SHIFT; + if (i == lastIdsI) { + if (lastIds == null) { + return 0; + } + if (add == null) { + return lastIds[index & BLOCK_MASK] & 0xFF; + } else { + return lastIds[index & BLOCK_MASK] & 0xFF + getAdd(index); + } + } + saveIds(); + byte[] compressed = ids[lastIdsI = i]; + if (compressed == null) { + lastIds = null; + return 0; + } + lastIds = MainUtil.decompress(compressed, lastIds, BLOCK_SIZE, compressionLevel); + if (add == null) { + return lastIds[index & BLOCK_MASK] & 0xFF; + } else { + return lastIds[index & BLOCK_MASK] & 0xFF + getAdd(index); + } + } + + int saves = 0; + + private void saveIds() { + if (saveIds && lastIds != null) { + ids[lastIdsI] = MainUtil.compress(lastIds, buffer, compressionLevel); + } + saveIds = false; + } + + private void saveDatas() { + if (saveDatas && lastDatas != null) { + datas[lastDatasI] = MainUtil.compress(lastDatas, buffer, compressionLevel); + } + saveDatas = false; + } + + private void saveAdd() { + if (saveAdd && lastAdd != null) { + add[lastAddI] = MainUtil.compress(lastAdd, buffer, compressionLevel); + } + saveAdd = false; + } + + public int getData(int index) { + int i = index >> BLOCK_SHIFT; + if (i == lastDatasI) { + if (lastDatas == null) { + return 0; + } + return lastDatas[index & BLOCK_MASK]; + } + saveDatas(); + byte[] compressed = datas[lastDatasI = i]; + if (compressed == null) { + lastDatas = null; + return 0; + } + lastDatas = MainUtil.decompress(compressed, lastDatas, BLOCK_SIZE, compressionLevel); + return lastDatas[index & BLOCK_MASK]; + } + + @Override + public void setDimensions(Vector dimensions) { + width = dimensions.getBlockX(); + height = dimensions.getBlockY(); + length = dimensions.getBlockZ(); + area = width * length; + } + + @Override + public Vector getDimensions() { + return new Vector(width, height, length); + } + + public int getAdd(int index) { + int i = index >> BLOCK_SHIFT; + if (i == lastAddI) { + if (lastAdd == null) { + return 0; + } + return lastAdd[index & BLOCK_MASK] & 0xFF; + } + saveAdd(); + byte[] compressed = add[lastAddI = i]; + if (compressed == null) { + lastAdd = null; + return 0; + } + lastAdd = MainUtil.decompress(compressed, lastAdd, BLOCK_SIZE, compressionLevel); + return lastAdd[index & BLOCK_MASK] & 0xFF; + } + + + private int lastI; + private int lastIMin; + private int lastIMax; + + public int getLocalIndex(int index) { + if (index < lastIMin || index > lastIMax) { + lastI = index >> BLOCK_SHIFT; + lastIMin = lastI << BLOCK_SHIFT; + lastIMax = lastIMin + BLOCK_MASK; + } + return lastI; + } + + @Override + public void setId(int index, int value) { + int i = getLocalIndex(index); + if (i != lastIdsI) { + saveIds(); + byte[] compressed = ids[lastIdsI = i]; + if (compressed != null) { + lastIds = MainUtil.decompress(compressed, lastIds, BLOCK_SIZE, compressionLevel); + } else { + lastIds = null; + } + } + if (lastIds == null) { + if (value == 0) { + return; + } + lastIds = new byte[BLOCK_SIZE]; + } + lastIds[index & BLOCK_MASK] = (byte) value; + saveIds = true; + } + + @Override + public void setData(int index, int value) { + int i = getLocalIndex(index); + if (i != lastDatasI) { + saveDatas(); + byte[] compressed = datas[lastDatasI = i]; + if (compressed != null) { + lastDatas = MainUtil.decompress(compressed, lastDatas, BLOCK_SIZE, compressionLevel); + } else { + lastDatas = null; + } + } + if (lastDatas == null) { + if (value == 0) { + return; + } + lastDatas = new byte[BLOCK_SIZE]; + } + lastDatas[index & BLOCK_MASK] = (byte) value; + saveDatas = true; + } + + @Override + public void setAdd(int index, int value) { + if (value == 0) { + return; + } + if (add == null) { + add = new byte[1 + (volume >> BLOCK_SHIFT)][]; + } + int i = index >> BLOCK_SHIFT; + if (i != lastAddI) { + saveAdd(); + byte[] compressed = add[lastAddI = i]; + if (compressed != null) { + lastAdd = MainUtil.decompress(compressed, lastAdd, BLOCK_SIZE, compressionLevel); + } else { + lastAdd = null; + } + } + if (lastAdd == null) { + if (value == 0) { + return; + } + lastAdd = new byte[BLOCK_SIZE]; + } + lastAdd[index & BLOCK_MASK] = (byte) value; + saveAdd = true; } private int ylast; @@ -48,28 +256,29 @@ public class MemoryOptimizedClipboard extends FaweClipboard { private int zlast; private int zlasti; + public int getIndex(int x, int y, int z) { + return x + ((ylast == y) ? ylasti : (ylasti = (ylast = y) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); + } + @Override public BaseBlock getBlock(int x, int y, int z) { - int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); - byte[] idArray = ids[i]; - if (idArray == null) { + int index = getIndex(x, y, z); + return getBlock(index); + } + + public BaseBlock getBlock(int index) { + int id = getId(index); + if (id == 0) { return FaweCache.CACHE_BLOCK[0]; } - int y2 = y & 0xF; - int id = idArray[y2] & 0xFF; BaseBlock block; - if (!FaweCache.hasData(id) || datas == null) { - block = FaweCache.CACHE_BLOCK[id << 4]; + if (FaweCache.hasData(id)) { + block = FaweCache.getBlock(id, getData(index)); } else { - byte[] dataArray = datas[i]; - if (dataArray == null) { - block = FaweCache.CACHE_BLOCK[id << 4]; - } else { - block = FaweCache.CACHE_BLOCK[(id << 4) + dataArray[y2]]; - } + block = FaweCache.getBlock(id, 0); } if (FaweCache.hasNBT(id)) { - CompoundTag nbt = nbtMap.get(new IntegerTrio(x, y, z)); + CompoundTag nbt = nbtMap.get(index); if (nbt != null) { block = new BaseBlock(block.getId(), block.getData()); block.setNbtData(nbt); @@ -80,244 +289,60 @@ public class MemoryOptimizedClipboard extends FaweClipboard { @Override public void forEach(final RunnableVal2 task, boolean air) { - BlockVector pos = new BlockVector(0, 0, 0); - int y1max = ((height + 15) >> 4); - for (int x = 0; x < width; x++) { - int i1 = x; +// Fawe.debug("Compressed: " + size() + "b | Uncompressed: " + (volume << 0x5) + "b"); + task.value1 = new Vector(0, 0, 0); + for (int y = 0, index = 0; y < height; y++) { for (int z = 0; z < length; z++) { - int i2 = i1 + z * width; - for (int y = 0; y < y1max; y++) { - int y1 = y << 4; - int i = i2 + y * area; - byte[] idArray = ids[i]; - if (idArray == null) { - if (!air) { - continue; - } - for (int y2 = 0; y2 < height; y2++) { - pos.x = x; - pos.z = z; - pos.y = y1 + y2; - int yy = y1 + y2; - if (yy >= height) { - break; - } - task.run(pos, EditSession.nullBlock); - } + for (int x = 0; x < width; x++, index++) { + task.value2 = getBlock(index); + if (!air && task.value2.getId() == 0) { continue; } - for (int y2 = 0; y2 < idArray.length; y2++) { - int yy = y1 + y2; - if (yy >= height) { - break; - } - int id = idArray[y2] & 0xFF; - if (id == 0 && !air) { - continue; - } - pos.x = x; - pos.z = z; - pos.y = y1 + y2; - BaseBlock block; - if (!FaweCache.hasData(id) || datas == null) { - block = FaweCache.CACHE_BLOCK[id << 4]; - } else { - byte[] dataArray = datas[i]; - if (dataArray == null) { - block = FaweCache.CACHE_BLOCK[id << 4]; - } else { - block = FaweCache.CACHE_BLOCK[(id << 4) + dataArray[y2]]; - } - } - if (FaweCache.hasNBT(id)) { - CompoundTag nbt = nbtMap.get(new IntegerTrio((int) pos.x, (int) pos.y, (int) pos.z)); - if (nbt != null) { - block = new BaseBlock(block.getId(), block.getData()); - block.setNbtData(nbt); - } - } - task.run(pos, block); - } + task.value1.x = x; + task.value1.y = y; + task.value1.z = z; + task.run(); } } } } + public int size() { + saveIds(); + saveDatas(); + int total = 0; + for (byte[] array : ids) { + if (array != null) { + total += array.length; + } + } + for (byte[] array : datas) { + if (array != null) { + total += array.length; + } + } + return total; + } + @Override public boolean setTile(int x, int y, int z, CompoundTag tag) { - nbtMap.put(new IntegerTrio(x, y, z), tag); + nbtMap.put(getIndex(x, y, z), tag); return true; } @Override public boolean setBlock(int x, int y, int z, BaseBlock block) { - final int id = block.getId(); - switch (id) { - case 0: { - int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); - byte[] idArray = ids[i]; - if (idArray != null) { - int y2 = y & 0xF; - idArray[y2] = 0; - } - return true; - } - case 54: - case 130: - case 142: - case 27: - case 137: - case 52: - case 154: - case 84: - case 25: - case 144: - case 138: - case 176: - case 177: - case 63: - case 119: - case 68: - case 323: - case 117: - case 116: - case 28: - case 66: - case 157: - case 61: - case 62: - case 140: - case 146: - case 149: - case 150: - case 158: - case 23: - case 123: - case 124: - case 29: - case 33: - case 151: - case 178: { - if (block.hasNbtData()) { - nbtMap.put(new IntegerTrio(x, y, z), block.getNbtData()); - } - int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); - int y2 = y & 0xF; - byte[] idArray = ids[i]; - if (idArray == null) { - idArray = new byte[heights[ylast >> 4]]; - ids[i] = idArray; - } - idArray[y2] = (byte) id; - if (FaweCache.hasData(id)) { - int data = block.getData(); - if (data == 0) { - return true; - } - if (datas == null) { - datas = new byte[area * ((height + 15) >> 4)][]; - } - byte[] dataArray = datas[i]; - if (dataArray == null) { - dataArray = datas[i] = new byte[heights[ylast >> 4]]; - } - dataArray[y2] = (byte) data; - } - return true; - } - case 2: - case 4: - case 13: - case 14: - case 15: - case 20: - case 21: - case 22: - case 30: - case 32: - case 37: - case 39: - case 40: - case 41: - case 42: - case 45: - case 46: - case 47: - case 48: - case 49: - case 51: - case 56: - case 57: - case 58: - case 60: - case 7: - case 11: - case 73: - case 74: - case 79: - case 80: - case 81: - case 82: - case 83: - case 85: - case 87: - case 88: - case 101: - case 102: - case 103: - case 110: - case 112: - case 113: - case 121: - case 122: - case 129: - case 133: - case 165: - case 166: - case 169: - case 170: - case 172: - case 173: - case 174: - case 188: - case 189: - case 190: - case 191: - case 192: { - int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); - int y2 = y & 0xF; - byte[] idArray = ids[i]; - if (idArray == null) { - idArray = new byte[heights[ylast >> 4]]; - ids[i] = idArray; - } - idArray[y2] = (byte) id; - return true; - } - default: { - int i = x + ((ylast == y) ? ylasti : (ylasti = ((ylast = y) >> 4) * area)) + ((zlast == z) ? zlasti : (zlasti = (zlast = z) * width)); - int y2 = y & 0xF; - byte[] idArray = ids[i]; - if (idArray == null) { - idArray = new byte[heights[ylast >> 4]]; - ids[i] = idArray; - } - idArray[y2] = (byte) id; - int data = block.getData(); - if (data == 0) { - return true; - } - if (datas == null) { - datas = new byte[area * ((height + 15) >> 4)][]; - } - byte[] dataArray = datas[i]; - if (dataArray == null) { - dataArray = datas[i] = new byte[heights[ylast >> 4]]; - } - dataArray[y2] = (byte) data; - return true; - } + return setBlock(getIndex(x, y, z), block); + } + + public boolean setBlock(int index, BaseBlock block) { + setId(index, (byte) block.getId()); + setData(index, (byte) block.getData()); + CompoundTag tile = block.getNbtData(); + if (tile != null) { + nbtMap.put(index, tile); } + return true; } @Override diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/LazyClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/ReadOnlyClipboard.java similarity index 74% rename from core/src/main/java/com/boydti/fawe/object/clipboard/LazyClipboard.java rename to core/src/main/java/com/boydti/fawe/object/clipboard/ReadOnlyClipboard.java index 8be6511e..bcf89aa2 100644 --- a/core/src/main/java/com/boydti/fawe/object/clipboard/LazyClipboard.java +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/ReadOnlyClipboard.java @@ -13,19 +13,19 @@ import com.sk89q.worldedit.regions.Region; import java.util.Iterator; import java.util.List; -public abstract class LazyClipboard extends FaweClipboard { +public abstract class ReadOnlyClipboard extends FaweClipboard { private final Region region; - public LazyClipboard(Region region) { + public ReadOnlyClipboard(Region region) { this.region = region; } - public static LazyClipboard of(final EditSession editSession, Region region) { + public static ReadOnlyClipboard of(final EditSession editSession, final Region region) { final Vector origin = region.getMinimumPoint(); final int mx = origin.getBlockX(); final int my = origin.getBlockY(); final int mz = origin.getBlockZ(); - return new LazyClipboard(region) { + return new ReadOnlyClipboard(region) { @Override public BaseBlock getBlock(int x, int y, int z) { return editSession.getLazyBlock(mx + x, my + y, mz + z); @@ -62,6 +62,16 @@ public abstract class LazyClipboard extends FaweClipboard { return region; } + @Override + public Vector getDimensions() { + return region.getMaximumPoint().subtract(region.getMinimumPoint()).add(1, 1, 1); + } + + @Override + public void setDimensions(Vector dimensions) { + throw new UnsupportedOperationException("Clipboard is immutable"); + } + @Override public abstract BaseBlock getBlock(int x, int y, int z); @@ -87,4 +97,19 @@ public abstract class LazyClipboard extends FaweClipboard { public boolean remove(ClipboardEntity clipboardEntity) { throw new UnsupportedOperationException("Clipboard is immutable"); } + + @Override + public void setId(int index, int id) { + throw new UnsupportedOperationException("Clipboard is immutable"); + } + + @Override + public void setData(int index, int data) { + throw new UnsupportedOperationException("Clipboard is immutable"); + } + + @Override + public void setAdd(int index, int id) { + throw new UnsupportedOperationException("Clipboard is immutable"); + } } diff --git a/core/src/main/java/com/boydti/fawe/object/schematic/FaweFormat.java b/core/src/main/java/com/boydti/fawe/object/schematic/FaweFormat.java index 87b8f7f8..9bd9ed77 100644 --- a/core/src/main/java/com/boydti/fawe/object/schematic/FaweFormat.java +++ b/core/src/main/java/com/boydti/fawe/object/schematic/FaweFormat.java @@ -93,7 +93,7 @@ public class FaweFormat implements ClipboardReader, ClipboardWriter { from = true; case 2: small = false; - case 1: { + case 1: { // Unknown size ox = in.readInt(); oz = in.readInt(); FaweOutputStream tmp = new FaweOutputStream(new ByteArrayOutputStream(Settings.HISTORY.BUFFER_SIZE)); diff --git a/core/src/main/java/com/boydti/fawe/object/schematic/Schematic.java b/core/src/main/java/com/boydti/fawe/object/schematic/Schematic.java index 6b88930e..012f406a 100644 --- a/core/src/main/java/com/boydti/fawe/object/schematic/Schematic.java +++ b/core/src/main/java/com/boydti/fawe/object/schematic/Schematic.java @@ -1,6 +1,6 @@ package com.boydti.fawe.object.schematic; -import com.boydti.fawe.object.clipboard.LazyClipboard; +import com.boydti.fawe.object.clipboard.ReadOnlyClipboard; import com.boydti.fawe.util.EditSessionBuilder; import com.sk89q.worldedit.EditSession; import com.sk89q.worldedit.MaxChangedBlocksException; @@ -42,7 +42,7 @@ public class Schematic { checkNotNull(region); checkNotNull(region.getWorld()); EditSession session = new EditSessionBuilder(region.getWorld()).allowedRegionsEverywhere().autoQueue(false).build(); - this.clipboard = new BlockArrayClipboard(region, LazyClipboard.of(session, region)); + this.clipboard = new BlockArrayClipboard(region, ReadOnlyClipboard.of(session, region)); } /** diff --git a/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweSchematicHandler.java b/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweSchematicHandler.java index 30d77cb6..d2925dd2 100644 --- a/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweSchematicHandler.java +++ b/core/src/main/java/com/boydti/fawe/regions/general/plot/FaweSchematicHandler.java @@ -3,7 +3,7 @@ package com.boydti.fawe.regions.general.plot; import com.boydti.fawe.FaweCache; import com.boydti.fawe.object.FaweQueue; import com.boydti.fawe.object.RunnableVal2; -import com.boydti.fawe.object.clipboard.LazyClipboard; +import com.boydti.fawe.object.clipboard.ReadOnlyClipboard; import com.boydti.fawe.util.EditSessionBuilder; import com.boydti.fawe.util.SetQueue; import com.boydti.fawe.util.TaskManager; @@ -65,7 +65,7 @@ public class FaweSchematicHandler extends SchematicHandler { final int my = pos1.getY(); final int mz = pos1.getZ(); - LazyClipboard clipboard = new LazyClipboard(region) { + ReadOnlyClipboard clipboard = new ReadOnlyClipboard(region) { @Override public BaseBlock getBlock(int x, int y, int z) { return editSession.getLazyBlock(mx + x, my + y, mz + z); diff --git a/core/src/main/java/com/boydti/fawe/util/MainUtil.java b/core/src/main/java/com/boydti/fawe/util/MainUtil.java index 9de4797b..7e54c8d8 100644 --- a/core/src/main/java/com/boydti/fawe/util/MainUtil.java +++ b/core/src/main/java/com/boydti/fawe/util/MainUtil.java @@ -39,6 +39,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -47,9 +48,12 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.lz4.LZ4InputStream; import net.jpountz.lz4.LZ4OutputStream; +import net.jpountz.lz4.LZ4Utils; public class MainUtil { /* @@ -102,37 +106,78 @@ public class MainUtil { } public static FaweOutputStream getCompressedOS(OutputStream os, int amount) throws IOException { + return getCompressedOS(os, amount, Settings.HISTORY.BUFFER_SIZE); + } + + private static final LZ4Factory FACTORY = LZ4Factory.fastestInstance(); + private static final LZ4Compressor COMPRESSOR = FACTORY.fastCompressor(); + private static final LZ4FastDecompressor DECOMPRESSOR = FACTORY.fastDecompressor(); + + public static int getMaxCompressedLength(int size) { + return LZ4Utils.maxCompressedLength(size); + } + + public static byte[] compress(byte[] bytes, byte[] buffer, int level) { + if (level == 0) { + return bytes; + } + LZ4Compressor compressor = level == 1 ? COMPRESSOR : FACTORY.highCompressor(level); + int decompressedLength = bytes.length; + if (buffer == null) { + int maxCompressedLength = compressor.maxCompressedLength(decompressedLength); + buffer = new byte[maxCompressedLength]; + } + int compressLen = compressor.compress(bytes, 0, decompressedLength, buffer, 0, buffer.length); + return Arrays.copyOf(buffer, compressLen); + } + + public static byte[] decompress(byte[] bytes, byte[] buffer, int length, int level) { + if (level == 0) { + return bytes; + } + if (buffer == null) { + buffer = new byte[length]; + } + DECOMPRESSOR.decompress(bytes, buffer); + return buffer; + } + + public static FaweOutputStream getCompressedOS(OutputStream os, int amount, int buffer) throws IOException { os.write((byte) amount); - os = new BufferedOutputStream(os, Settings.HISTORY.BUFFER_SIZE); + os = new BufferedOutputStream(os, buffer); if (amount == 0) { return new FaweOutputStream(os); } int gzipAmount = amount > 6 ? 1 : 0; for (int i = 0; i < gzipAmount; i++) { - os = new GZIPOutputStream(os, true); + os = new BufferedOutputStream(new GZIPOutputStream(os, buffer, true)); } LZ4Factory factory = LZ4Factory.fastestInstance(); int fastAmount = 1 + ((amount - 1) % 3); for (int i = 0; i < fastAmount; i++) { - os = new LZ4OutputStream(os, Settings.HISTORY.BUFFER_SIZE, factory.fastCompressor()); + os = new LZ4OutputStream(os, buffer, factory.fastCompressor()); } int highAmount = amount > 3 ? 1 : 0; for (int i = 0; i < highAmount; i++) { - os = new LZ4OutputStream(os, Settings.HISTORY.BUFFER_SIZE, factory.highCompressor()); + os = new LZ4OutputStream(os, buffer, factory.highCompressor()); } return new FaweOutputStream(os); } public static FaweInputStream getCompressedIS(InputStream is) throws IOException { + return getCompressedIS(is, Settings.HISTORY.BUFFER_SIZE); + } + + public static FaweInputStream getCompressedIS(InputStream is, int buffer) throws IOException { int amount = is.read(); - is = new BufferedInputStream(is, Settings.HISTORY.BUFFER_SIZE); + is = new BufferedInputStream(is, buffer); if (amount == 0) { return new FaweInputStream(is); } LZ4Factory factory = LZ4Factory.fastestInstance(); boolean gzip = amount > 6; if (gzip) { - is = new GZIPInputStream(is); + is = new BufferedInputStream(new GZIPInputStream(is, buffer)); } amount = (1 + ((amount - 1) % 3)) + (amount > 3 ? 1 : 0); for (int i = 0; i < amount; i++) { diff --git a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java index e97e57f2..a67dc58d 100644 --- a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java +++ b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java @@ -19,6 +19,7 @@ package com.sk89q.jnbt; +import com.boydti.fawe.jnbt.NBTStreamer; import com.boydti.fawe.object.RunnableVal2; import java.io.Closeable; import java.io.DataInputStream; @@ -135,8 +136,15 @@ public final class NBTInputStream implements Closeable { is.skip(length); return; } - for (int i = 0; i < length; i++) { - reader.run(i, is.readByte()); + if (reader instanceof NBTStreamer.ByteReader) { + NBTStreamer.ByteReader byteReader = (NBTStreamer.ByteReader) reader; + for (int i = 0; i < length; i++) { + byteReader.run(i, is.read()); + } + } else { + for (int i = 0; i < length; i++) { + reader.run(i, is.readByte()); + } } return; case NBTConstants.TYPE_LIST: diff --git a/core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java b/core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java index 8818613e..b1e4efff 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java @@ -22,7 +22,7 @@ package com.sk89q.worldedit.command; import com.boydti.fawe.FaweAPI; import com.boydti.fawe.config.BBC; import com.boydti.fawe.object.RunnableVal2; -import com.boydti.fawe.object.clipboard.LazyClipboard; +import com.boydti.fawe.object.clipboard.ReadOnlyClipboard; import com.boydti.fawe.util.ImgurUtility; import com.sk89q.minecraft.util.commands.Command; import com.sk89q.minecraft.util.commands.CommandException; @@ -111,7 +111,7 @@ public class ClipboardCommands { final int mx = origin.getBlockX(); final int my = origin.getBlockY(); final int mz = origin.getBlockZ(); - LazyClipboard lazyClipboard = new LazyClipboard(region) { + ReadOnlyClipboard lazyClipboard = new ReadOnlyClipboard(region) { @Override public BaseBlock getBlock(int x, int y, int z) { return editSession.getLazyBlock(mx + x, my + y, mz + z); diff --git a/core/src/main/java/com/sk89q/worldedit/command/SchematicCommands.java b/core/src/main/java/com/sk89q/worldedit/command/SchematicCommands.java index 8f281c58..14ae834d 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/SchematicCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/SchematicCommands.java @@ -46,8 +46,6 @@ import com.sk89q.worldedit.util.command.parametric.Optional; import com.sk89q.worldedit.util.io.file.FilenameException; import com.sk89q.worldedit.util.io.file.FilenameResolutionException; import com.sk89q.worldedit.world.registry.WorldData; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; @@ -116,7 +114,6 @@ public class SchematicCommands { } in = new FileInputStream(f); } - in = new BufferedInputStream(in); final ClipboardReader reader = format.getReader(in); final WorldData worldData = player.getWorld().getWorldData(); final Clipboard clipboard; @@ -183,16 +180,14 @@ public class SchematicCommands { target = clipboard; } - try (BufferedOutputStream bos = new BufferedOutputStream(fos)) { - try (ClipboardWriter writer = format.getWriter(bos)) { - if (writer instanceof StructureFormat) { - ((StructureFormat) writer).write(target, holder.getWorldData(), player.getName()); - } else { - writer.write(target, holder.getWorldData()); - } - log.info(player.getName() + " saved " + f.getCanonicalPath()); - BBC.SCHEMATIC_SAVED.send(player, filename); + try (ClipboardWriter writer = format.getWriter(fos)) { + if (writer instanceof StructureFormat) { + ((StructureFormat) writer).write(target, holder.getWorldData(), player.getName()); + } else { + writer.write(target, holder.getWorldData()); } + log.info(player.getName() + " saved " + f.getCanonicalPath()); + BBC.SCHEMATIC_SAVED.send(player, filename); } } } catch (IllegalArgumentException e) { diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/BlockArrayClipboard.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/BlockArrayClipboard.java index c39999ba..ea6870be 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/BlockArrayClipboard.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/BlockArrayClipboard.java @@ -50,20 +50,12 @@ import static com.google.common.base.Preconditions.checkNotNull; */ public class BlockArrayClipboard implements Clipboard { - private final Region region; - + private Region region; public FaweClipboard IMP; - - private final Vector size; - - + private Vector size; private int mx; private int my; private int mz; - - private int dx; - private int dxz; - private Vector origin; public BlockArrayClipboard(Region region) { @@ -99,7 +91,19 @@ public class BlockArrayClipboard implements Clipboard { checkNotNull(region); this.region = region.clone(); this.size = getDimensions(); + this.origin = region.getMinimumPoint(); + this.mx = origin.getBlockX(); + this.my = origin.getBlockY(); + this.mz = origin.getBlockZ(); this.IMP = clipboard; + } + + public void init(Region region, FaweClipboard fc) { + checkNotNull(region); + checkNotNull(fc); + this.region = region.clone(); + this.size = getDimensions(); + this.IMP = fc; this.origin = region.getMinimumPoint(); this.mx = origin.getBlockX(); this.my = origin.getBlockY(); diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java index 514138ab..39f90a7f 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java @@ -29,6 +29,8 @@ import com.boydti.fawe.util.MainUtil; import com.sk89q.jnbt.NBTConstants; import com.sk89q.jnbt.NBTInputStream; import com.sk89q.jnbt.NBTOutputStream; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; @@ -60,19 +62,21 @@ public enum ClipboardFormat { SCHEMATIC("mcedit", "mce", "schematic") { @Override public ClipboardReader getReader(InputStream inputStream) throws IOException { - NBTInputStream nbtStream = new NBTInputStream(new GZIPInputStream(inputStream)); + inputStream = new BufferedInputStream(inputStream); + NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(inputStream))); return new SchematicReader(nbtStream); } @Override public ClipboardWriter getWriter(OutputStream outputStream) throws IOException { + outputStream = new BufferedOutputStream(outputStream); GZIPOutputStream gzip; if (outputStream instanceof GZIPOutputStream) { gzip = (GZIPOutputStream) outputStream; } else { gzip = new GZIPOutputStream(outputStream, true); } - NBTOutputStream nbtStream = new NBTOutputStream(gzip); + NBTOutputStream nbtStream = new NBTOutputStream(new BufferedOutputStream(gzip)); return new SchematicWriter(nbtStream); } @@ -112,19 +116,21 @@ public enum ClipboardFormat { STRUCTURE("structure", "nbt") { @Override public ClipboardReader getReader(InputStream inputStream) throws IOException { - NBTInputStream nbtStream = new NBTInputStream(new GZIPInputStream(inputStream)); + inputStream = new BufferedInputStream(inputStream); + NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(inputStream))); return new StructureFormat(nbtStream); } @Override public ClipboardWriter getWriter(OutputStream outputStream) throws IOException { + outputStream = new BufferedOutputStream(outputStream); GZIPOutputStream gzip; if (outputStream instanceof GZIPOutputStream) { gzip = (GZIPOutputStream) outputStream; } else { gzip = new GZIPOutputStream(outputStream, true); } - NBTOutputStream nbtStream = new NBTOutputStream(gzip); + NBTOutputStream nbtStream = new NBTOutputStream(new BufferedOutputStream(gzip)); return new StructureFormat(nbtStream); } @@ -139,6 +145,9 @@ public enum ClipboardFormat { } }, + /** + * Isometric PNG writer + */ PNG("png", "image") { @Override public ClipboardReader getReader(InputStream inputStream) throws IOException { @@ -147,7 +156,7 @@ public enum ClipboardFormat { @Override public ClipboardWriter getWriter(OutputStream outputStream) throws IOException { - return new PNGWriter(outputStream); + return new PNGWriter(new BufferedOutputStream(outputStream)); } @Override diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java index c3d49c33..10dbccb4 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java @@ -19,35 +19,15 @@ package com.sk89q.worldedit.extent.clipboard.io; -import com.boydti.fawe.FaweCache; -import com.boydti.fawe.config.Settings; import com.boydti.fawe.jnbt.SchematicStreamer; -import com.sk89q.jnbt.ByteArrayTag; import com.sk89q.jnbt.CompoundTag; -import com.sk89q.jnbt.IntTag; -import com.sk89q.jnbt.ListTag; import com.sk89q.jnbt.NBTInputStream; -import com.sk89q.jnbt.NamedTag; -import com.sk89q.jnbt.ShortTag; -import com.sk89q.jnbt.StringTag; import com.sk89q.jnbt.Tag; -import com.sk89q.worldedit.BlockVector; -import com.sk89q.worldedit.Vector; -import com.sk89q.worldedit.blocks.BaseBlock; -import com.sk89q.worldedit.entity.BaseEntity; -import com.sk89q.worldedit.extent.clipboard.BlockArrayClipboard; import com.sk89q.worldedit.extent.clipboard.Clipboard; -import com.sk89q.worldedit.regions.CuboidRegion; -import com.sk89q.worldedit.regions.Region; -import com.sk89q.worldedit.util.Location; import com.sk89q.worldedit.world.registry.WorldData; -import com.sk89q.worldedit.world.storage.NBTConversions; import java.io.IOException; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -78,191 +58,7 @@ public class SchematicReader implements ClipboardReader { } public Clipboard read(WorldData data, UUID clipboardId) throws IOException { - if (Settings.CLIPBOARD.USE_DISK) { - return new SchematicStreamer(inputStream, clipboardId).getClipboard(); - } - // Schematic tag - NamedTag rootTag = inputStream.readNamedTag(); - if (!rootTag.getName().equals("Schematic")) { - throw new IOException("Tag 'Schematic' does not exist or is not first"); - } - CompoundTag schematicTag = (CompoundTag) rootTag.getTag(); - - // Check - Map schematic = schematicTag.getValue(); - if (!schematic.containsKey("Blocks")) { - throw new IOException("Schematic file is missing a 'Blocks' tag"); - } - - // Check type of Schematic - String materials = requireTag(schematic, "Materials", StringTag.class).getValue(); - if (!materials.equals("Alpha")) { - throw new IOException("Schematic file is not an Alpha schematic"); - } - - // ==================================================================== - // Metadata - // ==================================================================== - - Vector origin; - Region region; - - // Get information - short width = requireTag(schematic, "Width", ShortTag.class).getValue(); - short height = requireTag(schematic, "Height", ShortTag.class).getValue(); - short length = requireTag(schematic, "Length", ShortTag.class).getValue(); - - try { - int originX = requireTag(schematic, "WEOriginX", IntTag.class).getValue(); - int originY = requireTag(schematic, "WEOriginY", IntTag.class).getValue(); - int originZ = requireTag(schematic, "WEOriginZ", IntTag.class).getValue(); - Vector min = new Vector(originX, originY, originZ); - - int offsetX = requireTag(schematic, "WEOffsetX", IntTag.class).getValue(); - int offsetY = requireTag(schematic, "WEOffsetY", IntTag.class).getValue(); - int offsetZ = requireTag(schematic, "WEOffsetZ", IntTag.class).getValue(); - Vector offset = new Vector(offsetX, offsetY, offsetZ); - - origin = min.subtract(offset); - region = new CuboidRegion(min, min.add(width, height, length).subtract(Vector.ONE)); - } catch (IOException ignored) { - origin = new Vector(0, 0, 0); - region = new CuboidRegion(origin, origin.add(width, height, length).subtract(Vector.ONE)); - } - - // ==================================================================== - // Blocks - // ==================================================================== - - // Get blocks - byte[] blockId = requireTag(schematic, "Blocks", ByteArrayTag.class).getValue(); - byte[] blockData = requireTag(schematic, "Data", ByteArrayTag.class).getValue(); - byte[] addId = null; - - // We support 4096 block IDs using the same method as vanilla Minecraft, where - // the highest 4 bits are stored in a separate byte array. - if (schematic.containsKey("AddBlocks")) { - addId = requireTag(schematic, "AddBlocks", ByteArrayTag.class).getValue(); - } - - // Need to pull out tile entities - List tileEntities = requireTag(schematic, "TileEntities", ListTag.class).getValue(); - Map> tileEntitiesMap = new HashMap>(); - - for (Tag tag : tileEntities) { - if (!(tag instanceof CompoundTag)) continue; - CompoundTag t = (CompoundTag) tag; - - int x = 0; - int y = 0; - int z = 0; - - Map values = new HashMap(); - - for (Map.Entry entry : t.getValue().entrySet()) { - if (entry.getKey().equals("x")) { - if (entry.getValue() instanceof IntTag) { - x = ((IntTag) entry.getValue()).getValue(); - } - } else if (entry.getKey().equals("y")) { - if (entry.getValue() instanceof IntTag) { - y = ((IntTag) entry.getValue()).getValue(); - } - } else if (entry.getKey().equals("z")) { - if (entry.getValue() instanceof IntTag) { - z = ((IntTag) entry.getValue()).getValue(); - } - } - - values.put(entry.getKey(), entry.getValue()); - } - - BlockVector vec = new BlockVector(x, y, z); - tileEntitiesMap.put(vec, values); - } - - BlockArrayClipboard clipboard = new BlockArrayClipboard(region, clipboardId); - clipboard.setOrigin(origin); - - // Don't log a torrent of errors - int failedBlockSets = 0; - - Vector min = region.getMinimumPoint(); - int mx = min.getBlockX(); - int my = min.getBlockY(); - int mz = min.getBlockZ(); - - BlockVector pt = new BlockVector(0, 0, 0); - - int i; - for (int y = 0; y < height; y++) { - int yy = y + my; - final int i1 = y * width * length; - for (int z = 0; z < length; z++) { - int zz = z + mz; - final int i2 = (z * width) + i1; - for (int x = 0; x < width; x++) { - i = i2 + x; - int xx = x + mx; - int id = blockId[i] & 0xFF; - int db = blockData[i]; - if (addId != null) { - if ((i & 1) == 0) { - id += ((addId[i >> 1] & 0x0F) << 8); - } else { - id += ((addId[i >> 1] & 0xF0) << 4); - } - } - BaseBlock block = FaweCache.getBlock(id, db); - if (FaweCache.hasNBT(id)) { - pt.x = x; - pt.y = y; - pt.z = z; - if (tileEntitiesMap.containsKey(pt)) { - block = new BaseBlock(block.getId(), block.getData()); - block.setNbtData(new CompoundTag(tileEntitiesMap.get(pt))); - } - } - try { - clipboard.setBlock(xx, yy, zz, block); - } catch (Exception e) { - switch (failedBlockSets) { - case 0: - log.log(Level.WARNING, "Failed to set block on a Clipboard", e); - break; - case 1: - log.log(Level.WARNING, "Failed to set block on a Clipboard (again) -- no more messages will be logged", e); - break; - default: - } - failedBlockSets++; - } - } - } - } - - // ==================================================================== - // Entities - // ==================================================================== - - try { - List entityTags = requireTag(schematic, "Entities", ListTag.class).getValue(); - - for (Tag tag : entityTags) { - if (tag instanceof CompoundTag) { - CompoundTag compound = (CompoundTag) tag; - String id = compound.getString("id"); - Location location = NBTConversions.toLocation(clipboard, compound.getListTag("Pos"), compound.getListTag("Rotation")); - if (!id.isEmpty()) { - BaseEntity state = new BaseEntity(id, compound); - clipboard.createEntity(location, state); - } - } - } - } catch (IOException ignored) { // No entities? No problem - } - - return clipboard; + return new SchematicStreamer(inputStream, clipboardId).getClipboard(); } private static T requireTag(Map items, String key, Class expected) throws IOException { diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicWriter.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicWriter.java index f338fd82..7730ebca 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicWriter.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicWriter.java @@ -145,7 +145,7 @@ public class SchematicWriter implements ClipboardWriter { schematic.put("WEOffsetZ", new IntTag(offset.getBlockZ())); final byte[] blocks = new byte[width * height * length]; - byte[] addBlocks = null; + byte[] addBlocks; final byte[] blockData = new byte[width * height * length]; final List tileEntities = new ArrayList(); // Precalculate index vars diff --git a/core/src/main/java/net/jpountz/lz4/LZ4Utils.java b/core/src/main/java/net/jpountz/lz4/LZ4Utils.java index 5661c417..92c5851a 100644 --- a/core/src/main/java/net/jpountz/lz4/LZ4Utils.java +++ b/core/src/main/java/net/jpountz/lz4/LZ4Utils.java @@ -19,12 +19,12 @@ import static net.jpountz.lz4.LZ4Constants.HASH_LOG_64K; import static net.jpountz.lz4.LZ4Constants.HASH_LOG_HC; import static net.jpountz.lz4.LZ4Constants.MIN_MATCH; -enum LZ4Utils { +public enum LZ4Utils { ; private static final int MAX_INPUT_SIZE = 0x7E000000; - static int maxCompressedLength(int length) { + public static int maxCompressedLength(int length) { if (length < 0) { throw new IllegalArgumentException("length must be >= 0, got " + length); } else if (length >= MAX_INPUT_SIZE) { @@ -33,19 +33,19 @@ enum LZ4Utils { return length + length / 255 + 16; } - static int hash(int i) { + public static int hash(int i) { return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG); } - static int hash64k(int i) { + public static int hash64k(int i) { return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_64K); } - static int hashHC(int i) { + public static int hashHC(int i) { return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_HC); } - static class Match { + public static class Match { int start, ref, len; void fix(int correction) { @@ -59,7 +59,7 @@ enum LZ4Utils { } } - static void copyTo(Match m1, Match m2) { + public static void copyTo(Match m1, Match m2) { m2.len = m1.len; m2.start = m1.start; m2.ref = m1.ref;