+ * uint32_t mask_checksum(uint32_t x) {
+ * return ((x >> 15) | (x << 17)) + 0xa282ead8;
+ * }
+ *
+ */
+ public static int mask(int crc)
+ {
+ // Rotate right by 15 bits and add a constant.
+ return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
+ }
+
+
+ static final int readBytes(ReadableByteChannel source, ByteBuffer dest) throws IOException
+ {
+ // tells how many bytes to read.
+ final int expectedLength = dest.remaining();
+
+ int totalRead = 0;
+
+ // how many bytes were read.
+ int lastRead = source.read(dest);
+
+ totalRead = lastRead;
+
+ // if we did not read as many bytes as we had hoped, try reading again.
+ if (lastRead < expectedLength)
+ {
+ // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
+ while (dest.remaining() != 0 && lastRead != -1)
+ {
+ lastRead = source.read(dest);
+
+ // if we got EOF, do not add to total read.
+ if (lastRead != -1)
+ {
+ totalRead += lastRead;
+ }
+ }
+ }
+
+ if (totalRead > 0)
+ {
+ dest.limit(dest.position());
+ }
+ else
+ {
+ dest.position(dest.limit());
+ }
+
+ return totalRead;
+ }
+
+ static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) throws IOException
+ {
+ if (skip <= 0) {
+ return 0;
+ }
+
+ int toSkip = skip;
+ int skipped = 0;
+ while(toSkip > 0 && skipped != -1) {
+ buffer.clear();
+ if (toSkip < buffer.capacity()) {
+ buffer.limit(toSkip);
+ }
+
+ skipped = source.read(buffer);
+ if (skipped > 0) {
+ toSkip -= skipped;
+ }
+ }
+
+ buffer.clear();
+ return skip - toSkip;
+ }
+}
diff --git a/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java b/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
new file mode 100644
index 0000000..0c02018
--- /dev/null
+++ b/src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
@@ -0,0 +1,452 @@
+/*
+ * Created: Apr 15, 2013
+ */
+package org.xerial.snappy;
+
+import static java.lang.Math.min;
+import static org.xerial.snappy.SnappyFramed.*;
+import static org.xerial.snappy.SnappyFramedOutputStream.*;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Arrays;
+
+/**
+ * Implements the x-snappy-framed as an {@link InputStream} and
+ * {@link ReadableByteChannel}.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+public final class SnappyFramedInputStream extends InputStream implements
+ ReadableByteChannel {
+
+ private final ReadableByteChannel rbc;
+ private final ByteBuffer frameHeader;
+ private final boolean verifyChecksums;
+
+ /**
+ * A single frame read from the underlying {@link InputStream}.
+ */
+ private ByteBuffer input;
+
+ /**
+ * The decompressed data from {@link #input}.
+ */
+ private ByteBuffer uncompressedDirect;
+
+ /**
+ * Indicates if this instance has been closed.
+ */
+ private boolean closed;
+
+ /**
+ * Indicates if we have reached the EOF on {@link #in}.
+ */
+ private boolean eof;
+
+ /**
+ * The position in {@link buffer} to read to.
+ */
+ private int valid;
+
+ /**
+ * The next position to read from {@link #buffer}.
+ */
+ private int position;
+
+ /**
+ * Buffer contains a copy of the uncompressed data for the block.
+ */
+ private byte[] buffer;
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ *
+ * @param in
+ * the underlying input stream. Must not be {@code null}.
+ */
+ public SnappyFramedInputStream(InputStream in) throws IOException {
+ this(in, true);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ *
+ * @param in
+ * the underlying input stream. Must not be {@code null}.
+ * @param verifyChecksums
+ * if true, checksums in input stream will be verified
+ */
+ public SnappyFramedInputStream(InputStream in, boolean verifyChecksums)
+ throws IOException {
+ this(Channels.newChannel(in), verifyChecksums);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ *
+ * @param in
+ * the underlying readable channel. Must not be {@code null}.
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in)
+ throws IOException {
+ this(in, true);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ *
+ * @param in
+ * the underlying readable channel. Must not be {@code null}.
+ * @param verifyChecksums
+ * if true, checksums in input stream will be verified
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in,
+ boolean verifyChecksums) throws IOException {
+ if (in == null) {
+ throw new NullPointerException("in is null");
+ }
+
+ this.rbc = in;
+ this.verifyChecksums = verifyChecksums;
+
+ allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5);
+ this.frameHeader = ByteBuffer.allocate(4);
+
+ // stream must begin with stream header
+ final byte[] expectedHeader = HEADER_BYTES;
+ final byte[] actualHeader = new byte[expectedHeader.length];
+ final ByteBuffer actualBuffer = ByteBuffer.wrap(actualHeader);
+
+ // assume that if the input cannot read 4 bytes that something is
+ // wrong.
+ final int read = in.read(actualBuffer);
+ if (read < expectedHeader.length) {
+ throw new EOFException(
+ "encountered EOF while reading stream header");
+ }
+ if (!Arrays.equals(expectedHeader, actualHeader)) {
+ throw new IOException("invalid stream header");
+ }
+ }
+
+ /**
+ * @param size
+ */
+ private void allocateBuffersBasedOnSize(int size) {
+
+ input = ByteBuffer.allocateDirect(size);
+ final int maxCompressedLength = Snappy.maxCompressedLength(size);
+ uncompressedDirect = ByteBuffer.allocateDirect(maxCompressedLength);
+ buffer = new byte[maxCompressedLength];
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (closed) {
+ return -1;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+ return buffer[position++] & 0xFF;
+ }
+
+ @Override
+ public int read(byte[] output, int offset, int length) throws IOException {
+
+ if (output == null) {
+ throw new IllegalArgumentException("output is null");
+ }
+
+ if (offset < 0 || length < 0 || offset + length > output.length) {
+ throw new IllegalArgumentException("invalid offset [" + offset
+ + "] and length [" + length + ']');
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (length == 0) {
+ return 0;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+
+ final int size = min(length, available());
+ System.arraycopy(buffer, position, output, offset, size);
+ position += size;
+ return size;
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (closed) {
+ return 0;
+ }
+ return valid - position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isOpen() {
+ return !closed;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int read(ByteBuffer dst) throws IOException {
+
+ if (dst == null) {
+ throw new IllegalArgumentException("dst is null");
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (dst.remaining() == 0) {
+ return 0;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+
+ final int size = min(dst.remaining(), available());
+ dst.put(buffer, position, size);
+ position += size;
+ return size;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ rbc.close();
+ } finally {
+ if (!closed) {
+ closed = true;
+ }
+ }
+ }
+
+ static enum FrameAction {
+ RAW, SKIP, UNCOMPRESS;
+ }
+
+ public static final class FrameMetaData {
+ final int length;
+ final FrameAction frameAction;
+
+ /**
+ * @param frameAction
+ * @param length
+ */
+ public FrameMetaData(FrameAction frameAction, int length) {
+ super();
+ this.frameAction = frameAction;
+ this.length = length;
+ }
+ }
+
+ public static final class FrameData {
+ final int checkSum;
+ final int offset;
+
+ /**
+ * @param checkSum
+ * @param offset
+ */
+ public FrameData(int checkSum, int offset) {
+ super();
+ this.checkSum = checkSum;
+ this.offset = offset;
+ }
+ }
+
+ private boolean ensureBuffer() throws IOException {
+ if (available() > 0) {
+ return true;
+ }
+ if (eof) {
+ return false;
+ }
+
+ if (!readBlockHeader()) {
+ eof = true;
+ return false;
+ }
+
+ // get action based on header
+ final FrameMetaData frameMetaData = getFrameMetaData(frameHeader);
+
+ if (FrameAction.SKIP == frameMetaData.frameAction) {
+ SnappyFramed.skip(rbc, frameMetaData.length,
+ ByteBuffer.wrap(buffer));
+ return ensureBuffer();
+ }
+
+ if (frameMetaData.length > input.capacity()) {
+ allocateBuffersBasedOnSize(frameMetaData.length);
+ }
+
+ input.clear();
+ input.limit(frameMetaData.length);
+
+ final int actualRead = readBytes(rbc, input);
+ if (actualRead != frameMetaData.length) {
+ throw new EOFException("unexpectd EOF when reading frame");
+ }
+ input.flip();
+
+ final FrameData frameData = getFrameData(input);
+
+ if (FrameAction.UNCOMPRESS == frameMetaData.frameAction) {
+
+ input.position(frameData.offset);
+
+ final int uncompressedLength = Snappy.uncompressedLength(input);
+
+ if (uncompressedLength > uncompressedDirect.capacity()) {
+ uncompressedDirect = ByteBuffer
+ .allocateDirect(uncompressedLength);
+ buffer = new byte[Math.max(input.capacity(), uncompressedLength)];
+ }
+
+ uncompressedDirect.clear();
+
+ this.valid = Snappy.uncompress(input, uncompressedDirect);
+
+ uncompressedDirect.get(buffer, 0, valid);
+ this.position = 0;
+ } else {
+ // we need to start reading at the offset
+ input.position(frameData.offset);
+ this.position = 0;
+ this.valid = input.remaining();
+ this.input.get(buffer, 0, input.remaining());
+ }
+
+ if (verifyChecksums) {
+ final int actualCrc32c = SnappyFramed.maskedCrc32c(buffer,
+ position, valid - position);
+ if (frameData.checkSum != actualCrc32c) {
+ throw new IOException("Corrupt input: invalid checksum");
+ }
+ }
+
+ return true;
+ }
+
+ private boolean readBlockHeader() throws IOException {
+ frameHeader.clear();
+ int read = readBytes(rbc, frameHeader);
+
+ if (read == -1) {
+ return false;
+ }
+
+ if (read < frameHeader.capacity()) {
+ throw new EOFException("encountered EOF while reading block header");
+ }
+ frameHeader.flip();
+
+ return true;
+ }
+
+ /**
+ *
+ * @param frameHeader
+ * @return
+ * @throws IOException
+ */
+ private FrameMetaData getFrameMetaData(ByteBuffer frameHeader)
+ throws IOException {
+
+ assert frameHeader.hasArray();
+
+ final byte[] frameHeaderArray = frameHeader.array();
+
+ int length = (frameHeaderArray[1] & 0xFF);
+ length |= (frameHeaderArray[2] & 0xFF) << 8;
+ length |= (frameHeaderArray[3] & 0xFF) << 16;
+
+ int minLength = 0;
+ final FrameAction frameAction;
+ final int flag = frameHeaderArray[0] & 0xFF;
+ switch (flag) {
+ case COMPRESSED_DATA_FLAG:
+ frameAction = FrameAction.UNCOMPRESS;
+ minLength = 5;
+ break;
+ case UNCOMPRESSED_DATA_FLAG:
+ frameAction = FrameAction.RAW;
+ minLength = 5;
+ break;
+ case STREAM_IDENTIFIER_FLAG:
+ if (length != 6) {
+ throw new IOException(
+ "stream identifier chunk with invalid length: "
+ + length);
+ }
+ frameAction = FrameAction.SKIP;
+ minLength = 6;
+ break;
+ default:
+ // Reserved unskippable chunks (chunk types 0x02-0x7f)
+ if (flag <= 0x7f) {
+ throw new IOException("unsupported unskippable chunk: "
+ + Integer.toHexString(flag));
+ }
+
+ // all that is left is Reserved skippable chunks (chunk types
+ // 0x80-0xfe)
+ frameAction = FrameAction.SKIP;
+ minLength = 0;
+ }
+
+ if (length < minLength) {
+ throw new IOException("invalid length: " + length
+ + " for chunk flag: " + Integer.toHexString(flag));
+ }
+
+ return new FrameMetaData(frameAction, length);
+ }
+
+ /**
+ *
+ * @param content
+ * @return
+ * @throws IOException
+ */
+ private FrameData getFrameData(ByteBuffer content) throws IOException {
+ return new FrameData(getCrc32c(content), 4);
+ }
+
+ private int getCrc32c(ByteBuffer content) {
+
+ final int position = content.position();
+
+ return ((content.get(position + 3) & 0xFF) << 24)
+ | ((content.get(position + 2) & 0xFF) << 16)
+ | ((content.get(position + 1) & 0xFF) << 8)
+ | (content.get(position) & 0xFF);
+ }
+}
diff --git a/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java b/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java
new file mode 100644
index 0000000..221e242
--- /dev/null
+++ b/src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java
@@ -0,0 +1,317 @@
+/*
+ * Created: Apr 12, 2013
+ */
+package org.xerial.snappy;
+
+import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
+import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * Implements the x-snappy-framed as an {@link OutputStream} and
+ * {@link WritableByteChannel}.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+public final class SnappyFramedOutputStream extends OutputStream implements
+ WritableByteChannel {
+
+ /**
+ * The x-snappy-framed specification allows for a chunk size up to
+ * 16,777,211 bytes in length. However, it also goes on to state:
+ *
+ *
+ * We place an additional restriction that the uncompressed data in a chunk
+ * must be no longer than 65536 bytes. This allows consumers to easily use
+ * small fixed-size buffers.
+ *
+ *
- * (root class loader) -> [SnappyNativeLoader (load JNI code), SnappyNative (has native methods), SnappyNativeAPI, SnappyErrorCode] (injected by this method) - * | - * | - * (child class loader) -> Sees the above classes loaded by the root class loader. - * Then creates SnappyNativeAPI implementation by instantiating SnappyNaitive class. - *- * - * - *
- * (root class loader) -> [SnappyNativeLoader, SnappyNative ...] -> native code is loaded by once in this class loader - * | \ - * | (child2 class loader) - * (child1 class loader) - * - * child1 and child2 share the same SnappyNative code loaded by the root class loader. - *- * - * Note that Java's class loader first delegates the class lookup to its - * parent class loader. So once SnappyNativeLoader is loaded by the root - * class loader, no child class loader initialize SnappyNativeLoader again. - * - * @return - */ - static synchronized Object load() + static synchronized SnappyNative load() { if (api != null) return api; try { - if (!hasInjectedNativeLoader()) { - // Inject SnappyNativeLoader (src/main/resources/org/xerial/snappy/SnappyNativeLoader.bytecode) to the root class loader - Class< ? > nativeLoader = injectSnappyNativeLoader(); - // Load the JNI code using the injected loader - loadNativeLibrary(nativeLoader); - } + loadNativeLibrary(); + setApi(new SnappyNative()); isLoaded = true; - // Look up SnappyNative, injected to the root classloder, using reflection in order to avoid the initialization of SnappyNative class in this context class loader. - Object nativeCode = Class.forName("org.xerial.snappy.SnappyNative").newInstance(); - setApi(nativeCode); } catch (Exception e) { e.printStackTrace(); @@ -243,119 +156,40 @@ public class SnappyLoader } /** - * Inject SnappyNativeLoader class to the root class loader - * - * @return native code loader class initialized in the root class loader + * Load a native library of snappy-java */ - private static Class< ? > injectSnappyNativeLoader() { + private static void loadNativeLibrary() { - try { - // Use parent class loader to load SnappyNative, since Tomcat, which uses different class loaders for each webapps, cannot load JNI interface twice - - final String nativeLoaderClassName = "org.xerial.snappy.SnappyNativeLoader"; - ClassLoader rootClassLoader = getRootClassLoader(); - // Load a byte code - byte[] byteCode = getByteCode("/org/xerial/snappy/SnappyNativeLoader.bytecode"); - // In addition, we need to load the other dependent classes (e.g., SnappyNative and SnappyException) using the system class loader - final String[] classesToPreload = new String[] { "org.xerial.snappy.SnappyNativeAPI", - "org.xerial.snappy.SnappyNative", "org.xerial.snappy.SnappyErrorCode" }; - List
@@ -40,7 +39,7 @@ import java.nio.ByteBuffer; * @author leo * */ -public class SnappyNative implements SnappyNativeAPI +public class SnappyNative { public native String nativeLibraryVersion(); @@ -77,6 +76,8 @@ public class SnappyNative implements SnappyNativeAPI public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; + public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException; + public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; public void throw_error(int errorCode) throws IOException { diff --git a/src/main/java/org/xerial/snappy/SnappyNativeAPI.java b/src/main/java/org/xerial/snappy/SnappyNativeAPI.java deleted file mode 100755 index b6c92d0..0000000 --- a/src/main/java/org/xerial/snappy/SnappyNativeAPI.java +++ /dev/null @@ -1,82 +0,0 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// snappy-java Project -// -// SnappyNative.java -// Since: 2011/03/30 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * Internal only - Do not use this class. - * - * Interface to access the native code of Snappy. Although this class members - * are public, do not use them directly. Use {@link Snappy} API instead. - * - * - * @author leo - * - */ -public interface SnappyNativeAPI -{ - - public String nativeLibraryVersion(); - - // ------------------------------------------------------------------------ - // Generic compression/decompression routines. - // ------------------------------------------------------------------------ - public long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException; - public long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException; - - public int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, int outputOffset) - throws IOException; - - public int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException; - - public int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, - int outputOffset) throws IOException; - - public int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) - throws IOException; - - // Returns the maximal size of the compressed representation of - // input data that is "source_bytes" bytes in length; - public int maxCompressedLength(int source_bytes); - - // This operation takes O(1) time. - public int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException; - - public int uncompressedLength(Object input, int offset, int len) throws IOException; - - public long uncompressedLength(long inputAddr, long len) throws IOException; - - - public boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException; - - public boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; - - public void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; - - public void throw_error(int errorCode) throws IOException; - -} diff --git a/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so index 3299072..0aa1d90 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/arm/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/armhf/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/armhf/libsnappyjava.so index 23b37c6..1bf749c 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/armhf/libsnappyjava.so and b/src/main/resources/org/xerial/snappy/native/Linux/armhf/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/i386/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/i386/libsnappyjava.so deleted file mode 100755 index 0085465..0000000 Binary files a/src/main/resources/org/xerial/snappy/native/Linux/i386/libsnappyjava.so and /dev/null differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so new file mode 100755 index 0000000..4534c13 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/x86/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so new file mode 100755 index 0000000..20cf560 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib b/src/main/resources/org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib new file mode 100755 index 0000000..9daa6e6 Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib differ diff --git a/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib b/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib index 75f3e69..b9a2085 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib and b/src/main/resources/org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib differ diff --git a/src/main/resources/org/xerial/snappy/native/OpenBSD/x86/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/OpenBSD/x86/libsnappyjava.so new file mode 100755 index 0000000..deb885a Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/OpenBSD/x86/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/OpenBSD/x86_64/libsnappyjava.so b/src/main/resources/org/xerial/snappy/native/OpenBSD/x86_64/libsnappyjava.so new file mode 100755 index 0000000..e6e542b Binary files /dev/null and b/src/main/resources/org/xerial/snappy/native/OpenBSD/x86_64/libsnappyjava.so differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll index 5fc4252..5317765 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/amd64/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll similarity index 58% rename from src/main/resources/org/xerial/snappy/native/Windows/amd64/snappyjava.dll rename to src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll index a54064f..b367f2a 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/amd64/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll differ diff --git a/src/test/java/org/xerial/snappy/CalgaryTest.java b/src/test/java/org/xerial/snappy/CalgaryTest.java index 1f98af2..de72038 100755 --- a/src/test/java/org/xerial/snappy/CalgaryTest.java +++ b/src/test/java/org/xerial/snappy/CalgaryTest.java @@ -24,14 +24,20 @@ //-------------------------------------- package org.xerial.snappy; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.xerial.util.FileResource; import org.xerial.util.log.Logger; @@ -44,6 +50,9 @@ import org.xerial.util.log.Logger; public class CalgaryTest { private static Logger _logger = Logger.getLogger(CalgaryTest.class); + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); static byte[] readFile(String file) throws IOException { InputStream in = FileResource.find(CalgaryTest.class, file).openStream(); @@ -91,6 +100,83 @@ public class CalgaryTest } } + @Test + public void streamFramed() throws Exception { + for (String f : files) { + byte[] orig = readFile("testdata/calgary/" + f); + + ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); + SnappyFramedOutputStream out = new SnappyFramedOutputStream(compressedBuf); + out.write(orig); + out.close(); + + SnappyFramedInputStream in = new SnappyFramedInputStream(new ByteArrayInputStream(compressedBuf.toByteArray())); + + byte[] uncompressed = new byte[orig.length]; + int readBytes = readBytes(in, uncompressed, 0, orig.length); + + assertEquals(orig.length, readBytes); + assertArrayEquals(orig, uncompressed); + } + } + + @Test + public void streamFramedToFile() throws Exception { + for (String f : files) { + byte[] orig = readFile("testdata/calgary/" + f); + + final File tempFile = tempFolder.newFile(f); + final FileOutputStream compressedFOS = new FileOutputStream(tempFile); + try + { + SnappyFramedOutputStream out = new SnappyFramedOutputStream(compressedFOS); + out.write(orig); + out.close(); + } + finally + { + compressedFOS.close(); + } + + byte[] uncompressed = new byte[orig.length]; + + final FileInputStream compressedFIS = new FileInputStream(tempFile); + try + { + SnappyFramedInputStream in = new SnappyFramedInputStream(compressedFIS.getChannel()); + int readBytes = readBytes(in, uncompressed, 0, orig.length); + + assertEquals(orig.length, readBytes); + } + finally + { + compressedFIS.close(); + } + + assertArrayEquals(orig, uncompressed); + } + } + + @Test + public void streamFramedNoCRCVerify() throws Exception { + for (String f : files) { + byte[] orig = readFile("testdata/calgary/" + f); + + ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); + SnappyFramedOutputStream out = new SnappyFramedOutputStream(compressedBuf); + out.write(orig); + out.close(); + + SnappyFramedInputStream in = new SnappyFramedInputStream(new ByteArrayInputStream(compressedBuf.toByteArray()), false); + + byte[] uncompressed = new byte[orig.length]; + int readBytes = readBytes(in, uncompressed, 0, orig.length); + + assertEquals(orig.length, readBytes); + assertArrayEquals(orig, uncompressed); + } + } + @Test public void byteWiseRead() throws Exception { for (String f : files) { @@ -115,4 +201,28 @@ public class CalgaryTest } } + static final int readBytes(InputStream source, byte[] dest, int offset, int length) throws IOException + { + // how many bytes were read. + int lastRead = source.read(dest, offset, length); + + int totalRead = lastRead; + + // if we did not read as many bytes as we had hoped, try reading again. + if (lastRead < length) + { + // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading. + while (totalRead < length && lastRead != -1) + { + lastRead = source.read(dest, offset + totalRead, length - totalRead); + + // if we got EOF, do not add to total read. + if (lastRead != -1) + { + totalRead += lastRead; + } + } + } + return totalRead; + } } diff --git a/src/test/java/org/xerial/snappy/RandomGenerator.java b/src/test/java/org/xerial/snappy/RandomGenerator.java new file mode 100644 index 0000000..c75a8aa --- /dev/null +++ b/src/test/java/org/xerial/snappy/RandomGenerator.java @@ -0,0 +1,68 @@ +/* + * Created: Apr 15, 2013 + */ +package org.xerial.snappy; + +import java.util.Random; + +/** + * Generates random data with specific expected snappy performance characteristics. + * + *
+ * This has been copied from dain's snappy implementation.. + *
+ */ +public class RandomGenerator { + + public final byte[] data; + public int position; + + public RandomGenerator(double compressionRatio) { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd = new Random(301); + data = new byte[1048576 + 100]; + for (int i = 0; i < 1048576; i += 100) { + // Add a short fragment that is as compressible as specified ratio + System.arraycopy(compressibleData(rnd, compressionRatio, 100), 0, + data, i, 100); + } + } + + public int getNextPosition(int length) { + if (position + length > data.length) { + position = 0; + assert (length < data.length); + } + int result = position; + position += length; + return result; + } + + private static byte[] compressibleData(Random random, + double compressionRatio, int length) { + int raw = (int) (length * compressionRatio); + if (raw < 1) { + raw = 1; + } + byte[] rawData = generateRandomData(random, raw); + + // Duplicate the random data until we have filled "length" bytes + byte[] dest = new byte[length]; + for (int i = 0; i < length;) { + int chunkLength = Math.min(rawData.length, length - i); + System.arraycopy(rawData, 0, dest, i, chunkLength); + i += chunkLength; + } + return dest; + } + + private static byte[] generateRandomData(Random random, int length) { + byte[] rawData = new byte[length]; + for (int i = 0; i < rawData.length; i++) { + rawData[i] = (byte) random.nextInt(256); + } + return rawData; + } +} \ No newline at end of file diff --git a/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java b/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java new file mode 100644 index 0000000..decb43e --- /dev/null +++ b/src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java @@ -0,0 +1,300 @@ +/* + * Created: Mar 14, 2013 + */ +package org.xerial.snappy; + +import static org.xerial.snappy.SnappyFramed.*; +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + +import org.junit.Test; + +/** + * Tests the functionality of {@link org.xerial.snappy.SnappyFramedInputStream} + * and {@link org.xerial.snappy.SnappyFramedOutputStream}. + * + * @author Brett Okken + */ +public class SnappyFramedStreamTest { + + /** + * @throws IOException + */ + protected OutputStream createOutputStream(OutputStream target) + throws IOException { + return new SnappyFramedOutputStream(target); + } + + /** + * {@inheritDoc} + * + * @throws IOException + */ + protected InputStream createInputStream(InputStream source, + boolean verifyCheckSums) throws IOException { + return new SnappyFramedInputStream(source, verifyCheckSums); + } + + protected byte[] getMarkerFrame() { + return HEADER_BYTES; + } + + @Test + public void testSimple() throws Exception { + byte[] original = "aaaaaaaaaaaabbbbbbbaaaaaa".getBytes("utf-8"); + + byte[] compressed = compress(original); + byte[] uncompressed = uncompress(compressed); + + assertArrayEquals(uncompressed, original); + // 10 byte stream header, 4 byte block header, 4 byte crc, 19 bytes + assertEquals(compressed.length, 37); + + // stream header + assertArrayEquals(Arrays.copyOf(compressed, 10), HEADER_BYTES); + + // flag: compressed + assertEquals(toInt(compressed[10]), COMPRESSED_DATA_FLAG); + + // length: 23 = 0x000017 + assertEquals(toInt(compressed[11]), 0x17); + assertEquals(toInt(compressed[12]), 0x00); + assertEquals(toInt(compressed[13]), 0x00); + + // crc32c: 0x9274cda8 + assertEquals(toInt(compressed[17]), 0x92); + assertEquals(toInt(compressed[16]), 0x74); + assertEquals(toInt(compressed[15]), 0xCD); + assertEquals(toInt(compressed[14]), 0xA8); + } + + @Test + public void testUncompressable() throws Exception { + byte[] random = getRandom(1, 5000); + int crc32c = maskedCrc32c(random); + + byte[] compressed = compress(random); + byte[] uncompressed = uncompress(compressed); + + assertArrayEquals(uncompressed, random); + assertEquals(compressed.length, random.length + 10 + 4 + 4); + + // flag: uncompressed + assertEquals(toInt(compressed[10]), UNCOMPRESSED_DATA_FLAG); + + // length: 5004 = 0x138c + assertEquals(toInt(compressed[13]), 0x00); + assertEquals(toInt(compressed[12]), 0x13); + assertEquals(toInt(compressed[11]), 0x8c); + } + + @Test + public void testEmptyCompression() throws Exception { + byte[] empty = new byte[0]; + assertArrayEquals(compress(empty), HEADER_BYTES); + assertArrayEquals(uncompress(HEADER_BYTES), empty); + } + + @Test(expected = EOFException.class) + public void testShortBlockHeader() throws Exception { + uncompressBlock(new byte[] { 0 }); + } + + @Test(expected = EOFException.class) + public void testShortBlockData() throws Exception { + // flag = 0, size = 8, crc32c = 0, block data= [x, x] + uncompressBlock(new byte[] { 1, 8, 0, 0, 0, 0, 0, 0, 'x', 'x' }); + } + + @Test + public void testUnskippableChunkFlags() throws Exception { + for (int i = 2; i <= 0x7f; i++) { + try { + uncompressBlock(new byte[] { (byte) i, 5, 0, 0, 0, 0, 0, 0, 0 }); + fail("no exception thrown with flag: " + Integer.toHexString(i)); + } catch (IOException e) { + + } + } + } + + @Test + public void testSkippableChunkFlags() throws Exception { + for (int i = 0x80; i <= 0xfe; i++) { + try { + uncompressBlock(new byte[] { (byte) i, 5, 0, 0, 0, 0, 0, 0, 0 }); + } catch (IOException e) { + fail("exception thrown with flag: " + Integer.toHexString(i)); + } + } + } + + @Test(expected = IOException.class) + public void testInvalidBlockSizeZero() throws Exception { + // flag = '0', block size = 4, crc32c = 0 + uncompressBlock(new byte[] { 1, 4, 0, 0, 0, 0, 0, 0 }); + } + + @Test(expected = IOException.class) + public void testInvalidChecksum() throws Exception { + // flag = 0, size = 5, crc32c = 0, block data = [a] + uncompressBlock(new byte[] { 1, 5, 0, 0, 0, 0, 0, 0, 'a' }); + } + + @Test + public void testInvalidChecksumIgnoredWhenVerificationDisabled() + throws Exception { + // flag = 0, size = 4, crc32c = 0, block data = [a] + byte[] block = { 1, 5, 0, 0, 0, 0, 0, 0, 'a' }; + ByteArrayInputStream inputData = new ByteArrayInputStream( + blockToStream(block)); + assertArrayEquals(toByteArray(createInputStream(inputData, false)), + new byte[] { 'a' }); + } + + @Test + public void testLargerFrames_raw_() throws IOException { + final byte[] random = getRandom(0.5, 100000); + + final byte[] stream = new byte[HEADER_BYTES.length + 8 + random.length]; + System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); + + stream[10] = UNCOMPRESSED_DATA_FLAG; + + int length = random.length + 4; + stream[11] = (byte) length; + stream[12] = (byte) (length >>> 8); + stream[13] = (byte) (length >>> 16); + + int crc32c = maskedCrc32c(random); + stream[14] = (byte) crc32c; + stream[15] = (byte) (crc32c >>> 8); + stream[16] = (byte) (crc32c >>> 16); + stream[17] = (byte) (crc32c >>> 24); + + System.arraycopy(random, 0, stream, 18, random.length); + + final byte[] uncompressed = uncompress(stream); + + assertArrayEquals(random, uncompressed); + } + + @Test + public void testLargerFrames_compressed_() throws IOException { + final byte[] random = getRandom(0.5, 500000); + + final byte[] compressed = Snappy.compress(random); + + final byte[] stream = new byte[HEADER_BYTES.length + 8 + compressed.length]; + System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); + + stream[10] = COMPRESSED_DATA_FLAG; + + int length = compressed.length + 4; + stream[11] = (byte) length; + stream[12] = (byte) (length >>> 8); + stream[13] = (byte) (length >>> 16); + + int crc32c = maskedCrc32c(random); + stream[14] = (byte) crc32c; + stream[15] = (byte) (crc32c >>> 8); + stream[16] = (byte) (crc32c >>> 16); + stream[17] = (byte) (crc32c >>> 24); + + System.arraycopy(compressed, 0, stream, 18, compressed.length); + + final byte[] uncompressed = uncompress(stream); + + assertArrayEquals(random, uncompressed); + } + + @Test + public void testLargerFrames_compressed_smaller_raw_larger() + throws IOException { + final byte[] random = getRandom(0.5, 100000); + + final byte[] compressed = Snappy.compress(random); + + final byte[] stream = new byte[HEADER_BYTES.length + 8 + + compressed.length]; + System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); + + stream[10] = COMPRESSED_DATA_FLAG; + + int length = compressed.length + 4; + stream[11] = (byte) length; + stream[12] = (byte) (length >>> 8); + stream[13] = (byte) (length >>> 16); + + int crc32c = maskedCrc32c(random); + stream[14] = (byte) crc32c; + stream[15] = (byte) (crc32c >>> 8); + stream[16] = (byte) (crc32c >>> 16); + stream[17] = (byte) (crc32c >>> 24); + + System.arraycopy(compressed, 0, stream, 18, compressed.length); + + final byte[] uncompressed = uncompress(stream); + + assertArrayEquals(random, uncompressed); + } + + private byte[] uncompressBlock(byte[] block) throws IOException { + return uncompress(blockToStream(block)); + } + + private static byte[] blockToStream(byte[] block) { + byte[] stream = new byte[HEADER_BYTES.length + block.length]; + System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); + System.arraycopy(block, 0, stream, HEADER_BYTES.length, block.length); + return stream; + } + + + protected byte[] compress(byte[] original) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OutputStream snappyOut = createOutputStream(out); + snappyOut.write(original); + snappyOut.close(); + return out.toByteArray(); + } + + protected byte[] uncompress(byte[] compressed) throws IOException { + return toByteArray(createInputStream(new ByteArrayInputStream( + compressed), true)); + } + + private static byte[] toByteArray(InputStream createInputStream) throws IOException { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(64 * 1024); + + final byte[] buffer = new byte[8 * 1024]; + + int read; + while((read = createInputStream.read(buffer)) > 0) { + baos.write(buffer, 0, read); + } + + return baos.toByteArray(); + } + + static int toInt(byte value) { + return value & 0xFF; + } + + private byte[] getRandom(double compressionRatio, int length) { + RandomGenerator gen = new RandomGenerator( + compressionRatio); + gen.getNextPosition(length); + byte[] random = Arrays.copyOf(gen.data, length); + assertEquals(random.length, length); + return random; + } + +} diff --git a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java index 81ecd53..9bcc85f 100755 --- a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java +++ b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java @@ -24,23 +24,20 @@ //-------------------------------------- package org.xerial.snappy; -import static org.junit.Assert.*; - -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; - import org.codehaus.plexus.classworlds.ClassWorld; import org.codehaus.plexus.classworlds.realm.ClassRealm; import org.junit.Test; import org.xerial.util.FileResource; import org.xerial.util.log.Logger; +import java.io.*; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.fail; + public class SnappyLoaderTest { private static Logger _logger = Logger.getLogger(SnappyLoaderTest.class);