Fix higher compression values + message tweaks
This commit is contained in:
Jesse Boyd
parent
f9174ffb56
commit
a78a5e20ec
@ -26,25 +26,25 @@ public enum BBC {
|
||||
SCHEMATIC_PASTING("&7The schematic is pasting. This cannot be undone.", "Info"),
|
||||
LIGHTING_PROPOGATE_SELECTION("&7Lighting has been propogated in %s0 chunks. (Note: To remove light use //removelight)", "Info"),
|
||||
UPDATED_LIGHTING_SELECTION("&7Lighting has been updated in %s0 chunks. (It may take a second for the packets to send)", "Info"),
|
||||
SET_REGION("&7Selection set to your current WorldEdit region", "Info"),
|
||||
SET_REGION("&7Selection set to your current allowed region", "Info"),
|
||||
WORLDEDIT_COMMAND_LIMIT("&7Please wait until your current action completes", "Info"),
|
||||
WORLDEDIT_DELAYED("&7Please wait while we process your WorldEdit action...", "Info"),
|
||||
WORLDEDIT_DELAYED("&7Please wait while we process your FAWE action...", "Info"),
|
||||
WORLDEDIT_RUN("&7Apologies for the delay. Now executing: %s", "Info"),
|
||||
WORLDEDIT_COMPLETE("&7WorldEdit action completed.", "Info"),
|
||||
WORLDEDIT_COMPLETE("&7Edit completed.", "Info"),
|
||||
REQUIRE_SELECTION_IN_MASK("&7%s of your selection is not within your mask. You can only make edits within allowed regions.", "Info"),
|
||||
WORLDEDIT_VOLUME("&7You cannot select a volume of %current%. The maximum volume you can modify is %max%.", "Info"),
|
||||
WORLDEDIT_ITERATIONS("&7You cannot iterate %current% times. The maximum number of iterations allowed is %max%.", "Info"),
|
||||
WORLDEDIT_UNSAFE("&7Access to that command has been blocked", "Info"),
|
||||
WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe WorldEdit at %s0 by %s1", "Info"),
|
||||
WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe edit at %s0 by %s1", "Info"),
|
||||
WORLDEDIT_BYPASS("&7&oTo bypass your restrictions use &c/wea", "Info"),
|
||||
WORLDEDIT_EXTEND("&cYour WorldEdit may have extended outside your allowed region.", "Error"),
|
||||
WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled WorldEdit tips.", "Info"),
|
||||
WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled WorldEdit tips.", "Info"),
|
||||
WORLDEDIT_EXTEND("&cYour edit may have extended outside your allowed region.", "Error"),
|
||||
WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled FAWE tips.", "Info"),
|
||||
WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled FAWE tips.", "Info"),
|
||||
|
||||
WORLDEDIT_BYPASSED("&7Currently bypassing WorldEdit restriction.", "Info"),
|
||||
WORLDEDIT_UNMASKED("&6Your WorldEdit is now unrestricted.", "Info"),
|
||||
WORLDEDIT_BYPASSED("&7Currently bypassing FAWE restriction.", "Info"),
|
||||
WORLDEDIT_UNMASKED("&6Your FAWE edits are now unrestricted.", "Info"),
|
||||
|
||||
WORLDEDIT_RESTRICTED("&6Your WorldEdit is now restricted.", "Info"),
|
||||
WORLDEDIT_RESTRICTED("&6Your FAWE edits are now restricted.", "Info"),
|
||||
WORLDEDIT_OOM_ADMIN("&cPossible options:\n&8 - &7//fast\n&8 - &7Do smaller edits\n&8 - &7Allocate more memory\n&8 - &7Disable `max-memory-percent`", "Info"),
|
||||
COMPRESSED("History compressed. Saved ~ %s0b (%s1x smaller)", "Info"),
|
||||
|
||||
@ -231,7 +231,7 @@ public enum BBC {
|
||||
NO_PERM("&cYou are lacking the permission node: %s0", "Error"),
|
||||
SETTING_DISABLE("&cLacking setting: %s0","Error"),
|
||||
SCHEMATIC_NOT_FOUND("&cSchematic not found: &7%s0", "Error"),
|
||||
NO_REGION("&cYou have no current WorldEdit region", "Error"),
|
||||
NO_REGION("&cYou have no current allowed region", "Error"),
|
||||
NO_MASK("&cYou have no current mask set", "Error"),
|
||||
NOT_PLAYER("&cYou must be a player to perform this action!", "Error"),
|
||||
PLAYER_NOT_FOUND("&cPlayer not found:&7 %s0", "Error"),
|
||||
|
||||
@ -65,11 +65,12 @@ import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.Inflater;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import net.jpountz.lz4.LZ4BlockInputStream;
|
||||
import net.jpountz.lz4.LZ4BlockOutputStream;
|
||||
import net.jpountz.lz4.LZ4Compressor;
|
||||
import net.jpountz.lz4.LZ4Factory;
|
||||
import net.jpountz.lz4.LZ4FastDecompressor;
|
||||
import net.jpountz.lz4.LZ4InputStream;
|
||||
import net.jpountz.lz4.LZ4OutputStream;
|
||||
import net.jpountz.lz4.LZ4Utils;
|
||||
|
||||
public class MainUtil {
|
||||
@ -311,7 +312,7 @@ public class MainUtil {
|
||||
}
|
||||
|
||||
public static FaweOutputStream getCompressedOS(OutputStream os, int amount, int buffer) throws IOException {
|
||||
os.write((byte) -amount);
|
||||
os.write((byte) 9 + amount);
|
||||
os = new BufferedOutputStream(os, buffer);
|
||||
if (amount == 0) {
|
||||
return new FaweOutputStream(os);
|
||||
@ -324,14 +325,14 @@ public class MainUtil {
|
||||
LZ4Factory factory = LZ4Factory.fastestInstance();
|
||||
int fastAmount = 1 + ((amount - 1) % 3);
|
||||
for (int i = 0; i < fastAmount; i++) {
|
||||
os = new LZ4OutputStream(os, buffer, factory.fastCompressor());
|
||||
os = new LZ4BlockOutputStream(os, buffer, factory.fastCompressor());
|
||||
}
|
||||
int highAmount = amount > 3 ? 1 : 0;
|
||||
for (int i = 0; i < highAmount; i++) {
|
||||
if (amount == 9) {
|
||||
os = new LZ4OutputStream(os, buffer, factory.highCompressor(17));
|
||||
os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor(17));
|
||||
} else {
|
||||
os = new LZ4OutputStream(os, buffer, factory.highCompressor());
|
||||
os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor());
|
||||
}
|
||||
}
|
||||
return new FaweOutputStream(os);
|
||||
@ -342,14 +343,21 @@ public class MainUtil {
|
||||
}
|
||||
|
||||
public static FaweInputStream getCompressedIS(InputStream is, int buffer) throws IOException {
|
||||
int amount = (byte) is.read();
|
||||
int mode = (byte) is.read();
|
||||
is = new BufferedInputStream(is, buffer);
|
||||
if (amount == 0) {
|
||||
if (mode == 0) {
|
||||
return new FaweInputStream(is);
|
||||
}
|
||||
int amountAbs = Math.abs(amount);
|
||||
boolean legacy;
|
||||
if (mode > 9) {
|
||||
legacy = false;
|
||||
mode = -mode + 9;
|
||||
} else {
|
||||
legacy = true;
|
||||
}
|
||||
int amountAbs = Math.abs(mode);
|
||||
if (amountAbs > 6) {
|
||||
if (amount > 0) {
|
||||
if (mode > 0) {
|
||||
is = new BufferedInputStream(new GZIPInputStream(is, buffer));
|
||||
} else {
|
||||
is = new ZstdInputStream(is);
|
||||
@ -357,7 +365,11 @@ public class MainUtil {
|
||||
}
|
||||
amountAbs = (1 + ((amountAbs - 1) % 3)) + (amountAbs > 3 ? 1 : 0);
|
||||
for (int i = 0; i < amountAbs; i++) {
|
||||
is = new LZ4InputStream(is);
|
||||
if (legacy) {
|
||||
is = new LZ4InputStream(is);
|
||||
} else {
|
||||
is = new LZ4BlockInputStream(is);
|
||||
}
|
||||
}
|
||||
return new FaweInputStream(is);
|
||||
}
|
||||
|
||||
@ -94,7 +94,7 @@ public class ChunkCommands {
|
||||
@Command(
|
||||
aliases = { "delchunks" },
|
||||
usage = "",
|
||||
desc = "Delete chunks that your selection includes",
|
||||
desc = "Deprecated, use anvil commands",
|
||||
min = 0,
|
||||
max = 0
|
||||
)
|
||||
@ -114,7 +114,7 @@ public class ChunkCommands {
|
||||
out = new FileOutputStream("worldedit-delchunks.bat");
|
||||
OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8");
|
||||
writer.write("@ECHO off\r\n");
|
||||
writer.write("ECHO This batch file was generated by WorldEdit.\r\n");
|
||||
writer.write("ECHO This batch file was generated by FAWE.\r\n");
|
||||
writer.write("ECHO It contains a list of chunks that were in the selected region\r\n");
|
||||
writer.write("ECHO at the time that the /delchunks command was used. Run this file\r\n");
|
||||
writer.write("ECHO in order to delete the chunk files listed in this file.\r\n");
|
||||
@ -145,7 +145,7 @@ public class ChunkCommands {
|
||||
out = new FileOutputStream("worldedit-delchunks.sh");
|
||||
OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8");
|
||||
writer.write("#!/bin/bash\n");
|
||||
writer.write("echo This shell file was generated by WorldEdit.\n");
|
||||
writer.write("echo This shell file was generated by FAWE.\n");
|
||||
writer.write("echo It contains a list of chunks that were in the selected region\n");
|
||||
writer.write("echo at the time that the /delchunks command was used. Run this file\n");
|
||||
writer.write("echo in order to delete the chunk files listed in this file.\n");
|
||||
|
||||
@ -43,7 +43,7 @@ public class GeneralCommands {
|
||||
|
||||
@Command(
|
||||
aliases = { "/tips", "tips" },
|
||||
desc = "Toggle WorldEdit tips"
|
||||
desc = "Toggle FAWE tips"
|
||||
)
|
||||
public void tips(Player player, LocalSession session) throws WorldEditException {
|
||||
FawePlayer<Object> fp = FawePlayer.wrap(player);
|
||||
@ -57,7 +57,7 @@ public class GeneralCommands {
|
||||
@Command(
|
||||
aliases = { "/fast" },
|
||||
usage = "[on|off]",
|
||||
desc = "Toggles WorldEdit undo",
|
||||
desc = "Toggles FAWE undo",
|
||||
min = 0,
|
||||
max = 1
|
||||
)
|
||||
|
||||
@ -95,10 +95,10 @@ public class SnapshotUtilCommands {
|
||||
File dir = config.snapshotRepo.getDirectory();
|
||||
|
||||
try {
|
||||
logger.info("WorldEdit found no snapshots: looked in: "
|
||||
logger.info("FAWE found no snapshots: looked in: "
|
||||
+ dir.getCanonicalPath());
|
||||
} catch (IOException e) {
|
||||
logger.info("WorldEdit found no snapshots: looked in "
|
||||
logger.info("FAWE found no snapshots: looked in "
|
||||
+ "(NON-RESOLVABLE PATH - does it exist?): "
|
||||
+ dir.getPath());
|
||||
}
|
||||
|
||||
@ -66,22 +66,9 @@ public class WorldEditCommands {
|
||||
max = 0
|
||||
)
|
||||
public void version(Actor actor) throws WorldEditException {
|
||||
actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion());
|
||||
PlatformManager pm = we.getPlatformManager();
|
||||
actor.printDebug("------------------------------------");
|
||||
actor.printDebug("Platforms:");
|
||||
for (Platform platform : pm.getPlatforms()) {
|
||||
actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion()));
|
||||
}
|
||||
actor.printDebug("Capabilities:");
|
||||
for (Capability capability : Capability.values()) {
|
||||
Platform platform = pm.queryCapability(capability);
|
||||
actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE"));
|
||||
}
|
||||
actor.printDebug("------------------------------------");
|
||||
FaweVersion fVer = Fawe.get().getVersion();
|
||||
String fVerStr = fVer == null ? "unknown" : fVer.year + "." + fVer.month + "." + fVer.day + "-" + Integer.toHexString(fVer.hash) + "-" + fVer.build;
|
||||
actor.print(BBC.getPrefix() + "FAWE " + fVerStr);
|
||||
actor.print(BBC.getPrefix() + "FAWE " + fVerStr + " by Empire92");
|
||||
if (fVer != null) {
|
||||
actor.printDebug("------------------------------------");
|
||||
FaweVersion version = Fawe.get().getVersion();
|
||||
@ -99,8 +86,21 @@ public class WorldEditCommands {
|
||||
actor.printDebug(" - UPDATES: Latest Version");
|
||||
}
|
||||
actor.printDebug("------------------------------------");
|
||||
actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki");
|
||||
}
|
||||
actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion() + " by sk89q");
|
||||
PlatformManager pm = we.getPlatformManager();
|
||||
actor.printDebug("------------------------------------");
|
||||
actor.printDebug("Platforms:");
|
||||
for (Platform platform : pm.getPlatforms()) {
|
||||
actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion()));
|
||||
}
|
||||
actor.printDebug("Capabilities:");
|
||||
for (Capability capability : Capability.values()) {
|
||||
Platform platform = pm.queryCapability(capability);
|
||||
actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE"));
|
||||
}
|
||||
actor.printDebug("------------------------------------");
|
||||
actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki");
|
||||
}
|
||||
|
||||
@Command(
|
||||
@ -202,7 +202,7 @@ public class WorldEditCommands {
|
||||
@Command(
|
||||
aliases = { "help" },
|
||||
usage = "[<command>]",
|
||||
desc = "Displays help for WorldEdit commands",
|
||||
desc = "Displays help for FAWE commands",
|
||||
min = 0,
|
||||
max = -1
|
||||
)
|
||||
|
||||
@ -236,7 +236,7 @@ public final class CommandManager {
|
||||
.registerMethods(new ToolCommands(worldEdit))
|
||||
.registerMethods(new UtilityCommands(worldEdit))
|
||||
.group("worldedit", "we", "fawe")
|
||||
.describeAs("WorldEdit commands")
|
||||
.describeAs("FAWE commands")
|
||||
.registerMethods(new WorldEditCommands(worldEdit)).parent().group("schematic", "schem", "/schematic", "/schem")
|
||||
.describeAs("Schematic commands for saving/loading areas")
|
||||
.registerMethods(new SchematicCommands(worldEdit)).parent().group("snapshot", "snap")
|
||||
|
||||
235
core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java
Normal file
235
core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java
Normal file
@ -0,0 +1,235 @@
|
||||
package net.jpountz.lz4;
|
||||
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.zip.Checksum;
|
||||
import net.jpountz.util.SafeUtils;
|
||||
|
||||
|
||||
import static net.jpountz.lz4.LZ4BlockOutputStream.*;
|
||||
|
||||
/**
|
||||
* {@link InputStream} implementation to decode data written with
|
||||
* {@link LZ4BlockOutputStream}. This class is not thread-safe and does not
|
||||
* support {@link #mark(int)}/{@link #reset()}.
|
||||
* @see LZ4BlockOutputStream
|
||||
*/
|
||||
public final class LZ4BlockInputStream extends FilterInputStream {
|
||||
|
||||
private final LZ4FastDecompressor decompressor;
|
||||
private final Checksum checksum;
|
||||
private byte[] buffer;
|
||||
private byte[] compressedBuffer;
|
||||
private int originalLen;
|
||||
private int o;
|
||||
private boolean finished;
|
||||
|
||||
/**
|
||||
* Create a new {@link InputStream}.
|
||||
*
|
||||
* @param in the {@link InputStream} to poll
|
||||
* @param decompressor the {@link LZ4FastDecompressor decompressor} instance to
|
||||
* use
|
||||
* @param checksum the {@link Checksum} instance to use, must be
|
||||
* equivalent to the instance which has been used to
|
||||
* write the stream
|
||||
*/
|
||||
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor, Checksum checksum) {
|
||||
super(in);
|
||||
this.decompressor = decompressor;
|
||||
this.checksum = checksum;
|
||||
this.buffer = new byte[0];
|
||||
this.compressedBuffer = new byte[HEADER_LENGTH];
|
||||
o = originalLen = 0;
|
||||
finished = false;
|
||||
}
|
||||
|
||||
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor) {
|
||||
this(in, decompressor, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new instance which uses the fastest {@link LZ4FastDecompressor} available.
|
||||
* @see LZ4Factory#fastestInstance()
|
||||
* @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor)
|
||||
*/
|
||||
public LZ4BlockInputStream(InputStream in) {
|
||||
this(in, LZ4Factory.fastestInstance().fastDecompressor());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int available() throws IOException {
|
||||
return originalLen - o;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
if (o == originalLen) {
|
||||
refill();
|
||||
}
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
return buffer[o++] & 0xFF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
SafeUtils.checkRange(b, off, len);
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
if (o == originalLen) {
|
||||
refill();
|
||||
}
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
len = Math.min(len, originalLen - o);
|
||||
System.arraycopy(buffer, o, b, off, len);
|
||||
o += len;
|
||||
return len;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] b) throws IOException {
|
||||
return read(b, 0, b.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
if (o == originalLen) {
|
||||
refill();
|
||||
}
|
||||
if (finished) {
|
||||
return -1;
|
||||
}
|
||||
final int skipped = (int) Math.min(n, originalLen - o);
|
||||
o += skipped;
|
||||
return skipped;
|
||||
}
|
||||
|
||||
private void refill() throws IOException {
|
||||
readFully(compressedBuffer, HEADER_LENGTH);
|
||||
for (int i = 0; i < MAGIC_LENGTH; ++i) {
|
||||
if (compressedBuffer[i] != MAGIC[i]) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
}
|
||||
final int token = compressedBuffer[MAGIC_LENGTH] & 0xFF;
|
||||
final int compressionMethod = token & 0xF0;
|
||||
final int compressionLevel = COMPRESSION_LEVEL_BASE + (token & 0x0F);
|
||||
if (compressionMethod != COMPRESSION_METHOD_RAW && compressionMethod != COMPRESSION_METHOD_LZ4) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
final int compressedLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 1);
|
||||
originalLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 5);
|
||||
final int check = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 9);
|
||||
assert HEADER_LENGTH == MAGIC_LENGTH + 13;
|
||||
if (originalLen > 1 << compressionLevel
|
||||
|| originalLen < 0
|
||||
|| compressedLen < 0
|
||||
|| (originalLen == 0 && compressedLen != 0)
|
||||
|| (originalLen != 0 && compressedLen == 0)
|
||||
|| (compressionMethod == COMPRESSION_METHOD_RAW && originalLen != compressedLen)) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
if (originalLen == 0 && compressedLen == 0) {
|
||||
if (check != 0) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
finished = true;
|
||||
return;
|
||||
}
|
||||
if (buffer.length < originalLen) {
|
||||
buffer = new byte[Math.max(originalLen, buffer.length * 3 / 2)];
|
||||
}
|
||||
switch (compressionMethod) {
|
||||
case COMPRESSION_METHOD_RAW:
|
||||
readFully(buffer, originalLen);
|
||||
break;
|
||||
case COMPRESSION_METHOD_LZ4:
|
||||
if (compressedBuffer.length < originalLen) {
|
||||
compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)];
|
||||
}
|
||||
readFully(compressedBuffer, compressedLen);
|
||||
try {
|
||||
final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen);
|
||||
if (compressedLen != compressedLen2) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
} catch (LZ4Exception e) {
|
||||
throw new IOException("Stream is corrupted", e);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError();
|
||||
}
|
||||
if (checksum != null) {
|
||||
checksum.reset();
|
||||
checksum.update(buffer, 0, originalLen);
|
||||
if ((int) checksum.getValue() != check) {
|
||||
throw new IOException("Stream is corrupted");
|
||||
}
|
||||
}
|
||||
o = 0;
|
||||
}
|
||||
|
||||
private void readFully(byte[] b, int len) throws IOException {
|
||||
int read = 0;
|
||||
while (read < len) {
|
||||
final int r = in.read(b, read, len - read);
|
||||
if (r < 0) {
|
||||
throw new EOFException("Stream ended prematurely");
|
||||
}
|
||||
read += r;
|
||||
}
|
||||
assert len == read;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@SuppressWarnings("sync-override")
|
||||
@Override
|
||||
public void mark(int readlimit) {
|
||||
// unsupported
|
||||
}
|
||||
|
||||
@SuppressWarnings("sync-override")
|
||||
@Override
|
||||
public void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "(in=" + in
|
||||
+ ", decompressor=" + decompressor + ", checksum=" + checksum + ")";
|
||||
}
|
||||
|
||||
}
|
||||
256
core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java
Normal file
256
core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java
Normal file
@ -0,0 +1,256 @@
|
||||
package net.jpountz.lz4;
|
||||
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.FilterOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.zip.Checksum;
|
||||
|
||||
import net.jpountz.util.SafeUtils;
|
||||
|
||||
/**
|
||||
* Streaming LZ4.
|
||||
* <p>
|
||||
* This class compresses data into fixed-size blocks of compressed data.
|
||||
* @see LZ4BlockInputStream
|
||||
*/
|
||||
public final class LZ4BlockOutputStream extends FilterOutputStream {
|
||||
|
||||
static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' };
|
||||
static final int MAGIC_LENGTH = MAGIC.length;
|
||||
|
||||
static final int HEADER_LENGTH =
|
||||
MAGIC_LENGTH // magic bytes
|
||||
+ 1 // token
|
||||
+ 4 // compressed length
|
||||
+ 4 // decompressed length
|
||||
+ 4; // checksum
|
||||
|
||||
static final int COMPRESSION_LEVEL_BASE = 10;
|
||||
static final int MIN_BLOCK_SIZE = 64;
|
||||
static final int MAX_BLOCK_SIZE = 1 << (COMPRESSION_LEVEL_BASE + 0x0F);
|
||||
|
||||
static final int COMPRESSION_METHOD_RAW = 0x10;
|
||||
static final int COMPRESSION_METHOD_LZ4 = 0x20;
|
||||
|
||||
static final int DEFAULT_SEED = 0x9747b28c;
|
||||
|
||||
private static int compressionLevel(int blockSize) {
|
||||
if (blockSize < MIN_BLOCK_SIZE) {
|
||||
throw new IllegalArgumentException("blockSize must be >= " + MIN_BLOCK_SIZE + ", got " + blockSize);
|
||||
} else if (blockSize > MAX_BLOCK_SIZE) {
|
||||
throw new IllegalArgumentException("blockSize must be <= " + MAX_BLOCK_SIZE + ", got " + blockSize);
|
||||
}
|
||||
int compressionLevel = 32 - Integer.numberOfLeadingZeros(blockSize - 1); // ceil of log2
|
||||
assert (1 << compressionLevel) >= blockSize;
|
||||
assert blockSize * 2 > (1 << compressionLevel);
|
||||
compressionLevel = Math.max(0, compressionLevel - COMPRESSION_LEVEL_BASE);
|
||||
assert compressionLevel >= 0 && compressionLevel <= 0x0F;
|
||||
return compressionLevel;
|
||||
}
|
||||
|
||||
private final int blockSize;
|
||||
private final int compressionLevel;
|
||||
private final LZ4Compressor compressor;
|
||||
private final Checksum checksum;
|
||||
private final byte[] buffer;
|
||||
private final byte[] compressedBuffer;
|
||||
private final boolean syncFlush;
|
||||
private boolean finished;
|
||||
private int o;
|
||||
|
||||
/**
|
||||
* Create a new {@link OutputStream} with configurable block size. Large
|
||||
* blocks require more memory at compression and decompression time but
|
||||
* should improve the compression ratio.
|
||||
*
|
||||
* @param out the {@link OutputStream} to feed
|
||||
* @param blockSize the maximum number of bytes to try to compress at once,
|
||||
* must be >= 64 and <= 32 M
|
||||
* @param compressor the {@link LZ4Compressor} instance to use to compress
|
||||
* data
|
||||
* @param checksum the {@link Checksum} instance to use to check data for
|
||||
* integrity.
|
||||
* @param syncFlush true if pending data should also be flushed on {@link #flush()}
|
||||
*/
|
||||
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) {
|
||||
super(out);
|
||||
this.blockSize = blockSize;
|
||||
this.compressor = compressor;
|
||||
this.checksum = checksum;
|
||||
this.compressionLevel = compressionLevel(blockSize);
|
||||
this.buffer = new byte[blockSize];
|
||||
final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize);
|
||||
this.compressedBuffer = new byte[compressedBlockSize];
|
||||
this.syncFlush = syncFlush;
|
||||
o = 0;
|
||||
finished = false;
|
||||
System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH);
|
||||
}
|
||||
|
||||
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor) {
|
||||
this(out, blockSize, compressor, null, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new instance which compresses with the standard LZ4 compression
|
||||
* algorithm.
|
||||
* @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor)
|
||||
* @see LZ4Factory#fastCompressor()
|
||||
*/
|
||||
public LZ4BlockOutputStream(OutputStream out, int blockSize) {
|
||||
this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new instance which compresses into blocks of 64 KB.
|
||||
* @see #LZ4BlockOutputStream(OutputStream, int)
|
||||
*/
|
||||
public LZ4BlockOutputStream(OutputStream out) {
|
||||
this(out, 1 << 16);
|
||||
}
|
||||
|
||||
private void ensureNotFinished() {
|
||||
if (finished) {
|
||||
throw new IllegalStateException("This stream is already closed");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
ensureNotFinished();
|
||||
if (o == blockSize) {
|
||||
flushBufferedData();
|
||||
}
|
||||
buffer[o++] = (byte) b;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
SafeUtils.checkRange(b, off, len);
|
||||
ensureNotFinished();
|
||||
|
||||
while (o + len > blockSize) {
|
||||
final int l = blockSize - o;
|
||||
System.arraycopy(b, off, buffer, o, blockSize - o);
|
||||
o = blockSize;
|
||||
flushBufferedData();
|
||||
off += l;
|
||||
len -= l;
|
||||
}
|
||||
System.arraycopy(b, off, buffer, o, len);
|
||||
o += len;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
ensureNotFinished();
|
||||
write(b, 0, b.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (!finished) {
|
||||
finish();
|
||||
}
|
||||
if (out != null) {
|
||||
out.close();
|
||||
out = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void flushBufferedData() throws IOException {
|
||||
if (o == 0) {
|
||||
return;
|
||||
}
|
||||
final int check;
|
||||
if (checksum != null) {
|
||||
checksum.reset();
|
||||
checksum.update(buffer, 0, o);
|
||||
check = (int) checksum.getValue();
|
||||
} else {
|
||||
check = 0;
|
||||
}
|
||||
int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH);
|
||||
final int compressMethod;
|
||||
if (compressedLength >= o) {
|
||||
compressMethod = COMPRESSION_METHOD_RAW;
|
||||
compressedLength = o;
|
||||
System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o);
|
||||
} else {
|
||||
compressMethod = COMPRESSION_METHOD_LZ4;
|
||||
}
|
||||
|
||||
compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
|
||||
writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1);
|
||||
writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5);
|
||||
writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9);
|
||||
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
|
||||
out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength);
|
||||
o = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush this compressed {@link OutputStream}.
|
||||
*
|
||||
* If the stream has been created with <code>syncFlush=true</code>, pending
|
||||
* data will be compressed and appended to the underlying {@link OutputStream}
|
||||
* before calling {@link OutputStream#flush()} on the underlying stream.
|
||||
* Otherwise, this method just flushes the underlying stream, so pending
|
||||
* data might not be available for reading until {@link #finish()} or
|
||||
* {@link #close()} is called.
|
||||
*/
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
if (out != null) {
|
||||
if (syncFlush) {
|
||||
flushBufferedData();
|
||||
}
|
||||
out.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as {@link #close()} except that it doesn't close the underlying stream.
|
||||
* This can be useful if you want to keep on using the underlying stream.
|
||||
*/
|
||||
public void finish() throws IOException {
|
||||
ensureNotFinished();
|
||||
flushBufferedData();
|
||||
compressedBuffer[MAGIC_LENGTH] = (byte) (COMPRESSION_METHOD_RAW | compressionLevel);
|
||||
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 1);
|
||||
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 5);
|
||||
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 9);
|
||||
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
|
||||
out.write(compressedBuffer, 0, HEADER_LENGTH);
|
||||
finished = true;
|
||||
out.flush();
|
||||
}
|
||||
|
||||
private static void writeIntLE(int i, byte[] buf, int off) {
|
||||
buf[off++] = (byte) i;
|
||||
buf[off++] = (byte) (i >>> 8);
|
||||
buf[off++] = (byte) (i >>> 16);
|
||||
buf[off++] = (byte) (i >>> 24);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "(out=" + out + ", blockSize=" + blockSize
|
||||
+ ", compressor=" + compressor + ", checksum=" + checksum + ")";
|
||||
}
|
||||
|
||||
}
|
||||
@ -80,10 +80,6 @@ public class LZ4InputStream extends InputStream {
|
||||
return n - numBytesRemainingToSkip;
|
||||
}
|
||||
|
||||
public boolean hasBytesAvailableInDecompressedBuffer(int bytes) {
|
||||
return decompressedBufferPosition + bytes <= decompressedBufferLength;
|
||||
}
|
||||
|
||||
private boolean ensureBytesAvailableInDecompressedBuffer() throws IOException {
|
||||
while (decompressedBufferPosition >= decompressedBufferLength) {
|
||||
if (!fillBuffer()) {
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package net.jpountz.lz4;
|
||||
|
||||
import com.boydti.fawe.object.io.FastByteArrayOutputStream;
|
||||
import com.boydti.fawe.object.io.FastByteArraysInputStream;
|
||||
import com.boydti.fawe.util.MainUtil;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Random;
|
||||
@ -19,7 +19,7 @@ public class LZ4StreamTest {
|
||||
private Random rand;
|
||||
|
||||
private byte randomContent[];
|
||||
private byte compressedOutput[][];
|
||||
private byte compressedOutput[];
|
||||
|
||||
@Before
|
||||
public void setUp() throws IOException {
|
||||
@ -35,7 +35,7 @@ public class LZ4StreamTest {
|
||||
}
|
||||
|
||||
private void compressContent() throws IOException {
|
||||
FastByteArrayOutputStream compressedOutputStream = new FastByteArrayOutputStream();
|
||||
ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream();
|
||||
|
||||
LZ4OutputStream os = new LZ4OutputStream(compressedOutputStream);
|
||||
int currentContentPosition = 0;
|
||||
@ -69,13 +69,13 @@ public class LZ4StreamTest {
|
||||
|
||||
os.close();
|
||||
|
||||
compressedOutput = compressedOutputStream.toByteArrays();
|
||||
compressedOutput = compressedOutputStream.toByteArray();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void randomizedTest() throws IOException {
|
||||
try {
|
||||
InputStream is = new LZ4InputStream(new FastByteArraysInputStream(compressedOutput));
|
||||
InputStream is = new LZ4BlockInputStream(new ByteArrayInputStream(compressedOutput));
|
||||
|
||||
int currentContentPosition = 0;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user