- * The magic header is the following 8 bytes data: - * - *
- * -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 - *- * - * - * - * @author leo - * - */ -public class SnappyCodec -{ - public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 }; - public static final int MAGIC_LEN = MAGIC_HEADER.length; - public static final int HEADER_SIZE = MAGIC_LEN + 8; - public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0); - public static final int MAGIC_HEADER_TAIL = SnappyOutputStream.readInt(MAGIC_HEADER, 4); - - static { - assert(MAGIC_HEADER_HEAD < 0); - } - - - public static final int DEFAULT_VERSION = 1; - public static final int MINIMUM_COMPATIBLE_VERSION = 1; - - public final byte[] magic; - public final int version; - public final int compatibleVersion; - private final byte[] headerArray; - - private SnappyCodec(byte[] magic, int version, int compatibleVersion) { - this.magic = magic; - this.version = version; - this.compatibleVersion = compatibleVersion; - - ByteArrayOutputStream header = new ByteArrayOutputStream(HEADER_SIZE); - DataOutputStream d = new DataOutputStream(header); - try { - d.write(magic, 0, MAGIC_LEN); - d.writeInt(version); - d.writeInt(compatibleVersion); - d.close(); - } - catch(IOException e) { - throw new RuntimeException(e); - } - headerArray = header.toByteArray(); - } - - @Override - public String toString() { - return String.format("version:%d, compatible version:%d", version, compatibleVersion); - } - - public static int headerSize() { - return HEADER_SIZE; - } - - public int writeHeader(byte[] dst, int dstOffset) { - System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length); - return headerArray.length; - } - - public int writeHeader(OutputStream out) throws IOException { - out.write(headerArray, 0, headerArray.length); - return headerArray.length; - } - - public boolean isValidMagicHeader() { - return Arrays.equals(MAGIC_HEADER, magic); - } - - public static SnappyCodec readHeader(InputStream in) throws IOException { - DataInputStream d = new DataInputStream(in); - byte[] magic = new byte[MAGIC_LEN]; - d.readFully(magic, 0, MAGIC_LEN); - int version = d.readInt(); - int compatibleVersion = d.readInt(); - return new SnappyCodec(magic, version, compatibleVersion); - } - - public static SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION); - -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyCodec.java +// Since: 2011/04/03 14:50:20 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + +/** + * Preamble header for {@link SnappyOutputStream}. + * + *
+ * The magic header is the following 8 bytes data: + * + *
+ * -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 + *+ * + * + * + * @author leo + * + */ +public class SnappyCodec +{ + public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 }; + public static final int MAGIC_LEN = MAGIC_HEADER.length; + public static final int HEADER_SIZE = MAGIC_LEN + 8; + public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0); + public static final int MAGIC_HEADER_TAIL = SnappyOutputStream.readInt(MAGIC_HEADER, 4); + + static { + assert(MAGIC_HEADER_HEAD < 0); + } + + + public static final int DEFAULT_VERSION = 1; + public static final int MINIMUM_COMPATIBLE_VERSION = 1; + + public final byte[] magic; + public final int version; + public final int compatibleVersion; + private final byte[] headerArray; + + private SnappyCodec(byte[] magic, int version, int compatibleVersion) { + this.magic = magic; + this.version = version; + this.compatibleVersion = compatibleVersion; + + ByteArrayOutputStream header = new ByteArrayOutputStream(HEADER_SIZE); + DataOutputStream d = new DataOutputStream(header); + try { + d.write(magic, 0, MAGIC_LEN); + d.writeInt(version); + d.writeInt(compatibleVersion); + d.close(); + } + catch(IOException e) { + throw new RuntimeException(e); + } + headerArray = header.toByteArray(); + } + + @Override + public String toString() { + return String.format("version:%d, compatible version:%d", version, compatibleVersion); + } + + public static int headerSize() { + return HEADER_SIZE; + } + + public int writeHeader(byte[] dst, int dstOffset) { + System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length); + return headerArray.length; + } + + public int writeHeader(OutputStream out) throws IOException { + out.write(headerArray, 0, headerArray.length); + return headerArray.length; + } + + public boolean isValidMagicHeader() { + return Arrays.equals(MAGIC_HEADER, magic); + } + + public static SnappyCodec readHeader(InputStream in) throws IOException { + DataInputStream d = new DataInputStream(in); + byte[] magic = new byte[MAGIC_LEN]; + d.readFully(magic, 0, MAGIC_LEN); + int version = d.readInt(); + int compatibleVersion = d.readInt(); + return new SnappyCodec(magic, version, compatibleVersion); + } + + public static SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION); + +} diff --git a/src/main/java/org/xerial/snappy/SnappyError.java b/src/main/java/org/xerial/snappy/SnappyError.java index 0a3ab18..b366805 100755 --- a/src/main/java/org/xerial/snappy/SnappyError.java +++ b/src/main/java/org/xerial/snappy/SnappyError.java @@ -1,62 +1,62 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// XerialJ -// -// SnappyError.java -// Since: 2011/03/30 15:22:43 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -/** - * Used when serious errors (unchecked exception) are observed. - * - * @author leo - * - */ -public class SnappyError extends Error -{ - /** - * - */ - private static final long serialVersionUID = 1L; - - public final SnappyErrorCode errorCode; - - public SnappyError(SnappyErrorCode code) { - super(); - this.errorCode = code; - } - - public SnappyError(SnappyErrorCode code, Error e) { - super(e); - this.errorCode = code; - } - - public SnappyError(SnappyErrorCode code, String message) { - super(message); - this.errorCode = code; - } - - @Override - public String getMessage() { - return String.format("[%s] %s", errorCode.name(), super.getMessage()); - } - -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyError.java +// Since: 2011/03/30 15:22:43 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +/** + * Used when serious errors (unchecked exception) are observed. + * + * @author leo + * + */ +public class SnappyError extends Error +{ + /** + * + */ + private static final long serialVersionUID = 1L; + + public final SnappyErrorCode errorCode; + + public SnappyError(SnappyErrorCode code) { + super(); + this.errorCode = code; + } + + public SnappyError(SnappyErrorCode code, Error e) { + super(e); + this.errorCode = code; + } + + public SnappyError(SnappyErrorCode code, String message) { + super(message); + this.errorCode = code; + } + + @Override + public String getMessage() { + return String.format("[%s] %s", errorCode.name(), super.getMessage()); + } + +} diff --git a/src/main/java/org/xerial/snappy/SnappyErrorCode.java b/src/main/java/org/xerial/snappy/SnappyErrorCode.java index f1d8b6b..0dd6b4c 100755 --- a/src/main/java/org/xerial/snappy/SnappyErrorCode.java +++ b/src/main/java/org/xerial/snappy/SnappyErrorCode.java @@ -1,64 +1,64 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// XerialJ -// -// SnappyErrorCode.java -// Since: 2011/03/30 14:56:50 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -/** - * Error codes of snappy-java - * - * @author leo - * - */ -public enum SnappyErrorCode { - - // DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp - UNKNOWN(0), - FAILED_TO_LOAD_NATIVE_LIBRARY(1), - PARSING_ERROR(2), - NOT_A_DIRECT_BUFFER(3), - OUT_OF_MEMORY(4), - FAILED_TO_UNCOMPRESS(5), - EMPTY_INPUT(6), - INCOMPATIBLE_VERSION(7), - INVALID_CHUNK_SIZE(8) - ; - - public final int id; - - private SnappyErrorCode(int id) { - this.id = id; - } - - public static SnappyErrorCode getErrorCode(int id) { - for (SnappyErrorCode code : SnappyErrorCode.values()) { - if (code.id == id) - return code; - } - return UNKNOWN; - } - - public static String getErrorMessage(int id) { - return getErrorCode(id).name(); - } -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyErrorCode.java +// Since: 2011/03/30 14:56:50 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +/** + * Error codes of snappy-java + * + * @author leo + * + */ +public enum SnappyErrorCode { + + // DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp + UNKNOWN(0), + FAILED_TO_LOAD_NATIVE_LIBRARY(1), + PARSING_ERROR(2), + NOT_A_DIRECT_BUFFER(3), + OUT_OF_MEMORY(4), + FAILED_TO_UNCOMPRESS(5), + EMPTY_INPUT(6), + INCOMPATIBLE_VERSION(7), + INVALID_CHUNK_SIZE(8) + ; + + public final int id; + + private SnappyErrorCode(int id) { + this.id = id; + } + + public static SnappyErrorCode getErrorCode(int id) { + for (SnappyErrorCode code : SnappyErrorCode.values()) { + if (code.id == id) + return code; + } + return UNKNOWN; + } + + public static String getErrorMessage(int id) { + return getErrorCode(id).name(); + } +} diff --git a/src/main/java/org/xerial/snappy/SnappyException.java b/src/main/java/org/xerial/snappy/SnappyException.java index 81c08a5..e2b323d 100755 --- a/src/main/java/org/xerial/snappy/SnappyException.java +++ b/src/main/java/org/xerial/snappy/SnappyException.java @@ -1,74 +1,74 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// XerialJ -// -// SnappyException.java -// Since: 2011/03/30 14:56:14 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -import java.io.IOException; - -/** - * Exception in snappy-java - * - * @deprecated Snappy-java now uses {@link IOException} - * @author leo - * - */ -@Deprecated -public class SnappyException extends Exception -{ - private static final long serialVersionUID = 1L; - - public final SnappyErrorCode errorCode; - - public SnappyException(int code) { - this(SnappyErrorCode.getErrorCode(code)); - } - - public SnappyException(SnappyErrorCode errorCode) { - super(); - this.errorCode = errorCode; - } - - public SnappyException(SnappyErrorCode errorCode, Exception e) { - super(e); - this.errorCode = errorCode; - } - - public SnappyException(SnappyErrorCode errorCode, String message) { - super(message); - this.errorCode = errorCode; - } - - public SnappyErrorCode getErrorCode() { - return errorCode; - } - - public static void throwException(int errorCode) throws SnappyException { - throw new SnappyException(errorCode); - } - - @Override - public String getMessage() { - return String.format("[%s] %s", errorCode.name(), super.getMessage()); - } -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyException.java +// Since: 2011/03/30 14:56:14 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import java.io.IOException; + +/** + * Exception in snappy-java + * + * @deprecated Snappy-java now uses {@link IOException} + * @author leo + * + */ +@Deprecated +public class SnappyException extends Exception +{ + private static final long serialVersionUID = 1L; + + public final SnappyErrorCode errorCode; + + public SnappyException(int code) { + this(SnappyErrorCode.getErrorCode(code)); + } + + public SnappyException(SnappyErrorCode errorCode) { + super(); + this.errorCode = errorCode; + } + + public SnappyException(SnappyErrorCode errorCode, Exception e) { + super(e); + this.errorCode = errorCode; + } + + public SnappyException(SnappyErrorCode errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public SnappyErrorCode getErrorCode() { + return errorCode; + } + + public static void throwException(int errorCode) throws SnappyException { + throw new SnappyException(errorCode); + } + + @Override + public String getMessage() { + return String.format("[%s] %s", errorCode.name(), super.getMessage()); + } +} diff --git a/src/main/java/org/xerial/snappy/SnappyFramed.java b/src/main/java/org/xerial/snappy/SnappyFramed.java index 2925945..99b04e4 100644 --- a/src/main/java/org/xerial/snappy/SnappyFramed.java +++ b/src/main/java/org/xerial/snappy/SnappyFramed.java @@ -1,186 +1,186 @@ -/* - * Created: Apr 12, 2013 - */ -package org.xerial.snappy; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Constants and utilities for implementing x-snappy-framed. - * - * @author Brett Okken - * @since 1.1.0 - */ -final class SnappyFramed { - public static final int COMPRESSED_DATA_FLAG = 0x00; - - public static final int UNCOMPRESSED_DATA_FLAG = 0x01; - - public static final int STREAM_IDENTIFIER_FLAG = 0xff; - - private static final int MASK_DELTA = 0xa282ead8; - - /** - * Sun specific mechanisms to clean up resources associated with direct byte buffers. - */ - @SuppressWarnings("unchecked") - private static final Class SUN_DIRECT_BUFFER = (Class) lookupClassQuietly("sun.nio.ch.DirectBuffer"); - private static final Method SUN_BUFFER_CLEANER; - private static final Method SUN_CLEANER_CLEAN; - - static - { - Method bufferCleaner = null; - Method cleanerClean = null; - try { - //operate under the assumption that if the sun direct buffer class exists, - //all of the sun classes exist - if (SUN_DIRECT_BUFFER != null) { - bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[])null); - Class cleanClazz = lookupClassQuietly("sun.misc.Cleaner"); - cleanerClean = cleanClazz.getMethod("clean", (Class[])null); - } - } catch(Throwable t) { - Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t); - } - SUN_BUFFER_CLEANER = bufferCleaner; - SUN_CLEANER_CLEAN = cleanerClean; - } - - /** - * The header consists of the stream identifier flag, 3 bytes indicating a - * length of 6, and "sNaPpY" in ASCII. - */ - public static final byte[] HEADER_BYTES = new byte[] { - (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, - 0x50, 0x70, 0x59 }; - - public static int maskedCrc32c(byte[] data) - { - return maskedCrc32c(data, 0, data.length); - } - - public static int maskedCrc32c(byte[] data, int offset, int length) - { - final PureJavaCrc32C crc32c = new PureJavaCrc32C(); - crc32c.update(data, offset, length); - return mask(crc32c.getIntegerValue()); - } - - /** - * Checksums are not stored directly, but masked, as checksumming data and - * then its own checksum can be problematic. The masking is the same as used - * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant - * 0xa282ead8 (using wraparound as normal for unsigned integers). This is - * equivalent to the following C code: - * - *
- * uint32_t mask_checksum(uint32_t x) {
- * return ((x >> 15) | (x << 17)) + 0xa282ead8;
- * }
- *
- */
- public static int mask(int crc)
- {
- // Rotate right by 15 bits and add a constant.
- return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
- }
-
-
- static final int readBytes(ReadableByteChannel source, ByteBuffer dest) throws IOException
- {
- // tells how many bytes to read.
- final int expectedLength = dest.remaining();
-
- int totalRead = 0;
-
- // how many bytes were read.
- int lastRead = source.read(dest);
-
- totalRead = lastRead;
-
- // if we did not read as many bytes as we had hoped, try reading again.
- if (lastRead < expectedLength)
- {
- // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
- while (dest.remaining() != 0 && lastRead != -1)
- {
- lastRead = source.read(dest);
-
- // if we got EOF, do not add to total read.
- if (lastRead != -1)
- {
- totalRead += lastRead;
- }
- }
- }
-
- if (totalRead > 0)
- {
- dest.limit(dest.position());
- }
- else
- {
- dest.position(dest.limit());
- }
-
- return totalRead;
- }
-
- static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) throws IOException
- {
- if (skip <= 0) {
- return 0;
- }
-
- int toSkip = skip;
- int skipped = 0;
- while(toSkip > 0 && skipped != -1) {
- buffer.clear();
- if (toSkip < buffer.capacity()) {
- buffer.limit(toSkip);
- }
-
- skipped = source.read(buffer);
- if (skipped > 0) {
- toSkip -= skipped;
- }
- }
-
- buffer.clear();
- return skip - toSkip;
- }
-
- private static Class lookupClassQuietly(String name) {
- try {
- return SnappyFramed.class.getClassLoader().loadClass(name);
- } catch (Throwable t) {
- Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t);
- }
-
- return null;
- }
-
- /**
- * Provides jvm implementation specific operation to aggressively release resources associated with buffer.
- * @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}.
- */
- static void releaseDirectByteBuffer(ByteBuffer buffer)
- {
- assert buffer != null && buffer.isDirect();
-
- if (SUN_DIRECT_BUFFER != null && SUN_DIRECT_BUFFER.isAssignableFrom(buffer.getClass())) {
- try {
- Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null);
- SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null);
- } catch (Throwable t) {
- Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t);
- }
- }
- }
-}
+/*
+ * Created: Apr 12, 2013
+ */
+package org.xerial.snappy;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Constants and utilities for implementing x-snappy-framed.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+final class SnappyFramed {
+ public static final int COMPRESSED_DATA_FLAG = 0x00;
+
+ public static final int UNCOMPRESSED_DATA_FLAG = 0x01;
+
+ public static final int STREAM_IDENTIFIER_FLAG = 0xff;
+
+ private static final int MASK_DELTA = 0xa282ead8;
+
+ /**
+ * Sun specific mechanisms to clean up resources associated with direct byte buffers.
+ */
+ @SuppressWarnings("unchecked")
+ private static final Class SUN_DIRECT_BUFFER = (Class) lookupClassQuietly("sun.nio.ch.DirectBuffer");
+ private static final Method SUN_BUFFER_CLEANER;
+ private static final Method SUN_CLEANER_CLEAN;
+
+ static
+ {
+ Method bufferCleaner = null;
+ Method cleanerClean = null;
+ try {
+ //operate under the assumption that if the sun direct buffer class exists,
+ //all of the sun classes exist
+ if (SUN_DIRECT_BUFFER != null) {
+ bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[])null);
+ Class cleanClazz = lookupClassQuietly("sun.misc.Cleaner");
+ cleanerClean = cleanClazz.getMethod("clean", (Class[])null);
+ }
+ } catch(Throwable t) {
+ Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t);
+ }
+ SUN_BUFFER_CLEANER = bufferCleaner;
+ SUN_CLEANER_CLEAN = cleanerClean;
+ }
+
+ /**
+ * The header consists of the stream identifier flag, 3 bytes indicating a
+ * length of 6, and "sNaPpY" in ASCII.
+ */
+ public static final byte[] HEADER_BYTES = new byte[] {
+ (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
+ 0x50, 0x70, 0x59 };
+
+ public static int maskedCrc32c(byte[] data)
+ {
+ return maskedCrc32c(data, 0, data.length);
+ }
+
+ public static int maskedCrc32c(byte[] data, int offset, int length)
+ {
+ final PureJavaCrc32C crc32c = new PureJavaCrc32C();
+ crc32c.update(data, offset, length);
+ return mask(crc32c.getIntegerValue());
+ }
+
+ /**
+ * Checksums are not stored directly, but masked, as checksumming data and
+ * then its own checksum can be problematic. The masking is the same as used
+ * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
+ * 0xa282ead8 (using wraparound as normal for unsigned integers). This is
+ * equivalent to the following C code:
+ *
+ *
+ * uint32_t mask_checksum(uint32_t x) {
+ * return ((x >> 15) | (x << 17)) + 0xa282ead8;
+ * }
+ *
+ */
+ public static int mask(int crc)
+ {
+ // Rotate right by 15 bits and add a constant.
+ return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
+ }
+
+
+ static final int readBytes(ReadableByteChannel source, ByteBuffer dest) throws IOException
+ {
+ // tells how many bytes to read.
+ final int expectedLength = dest.remaining();
+
+ int totalRead = 0;
+
+ // how many bytes were read.
+ int lastRead = source.read(dest);
+
+ totalRead = lastRead;
+
+ // if we did not read as many bytes as we had hoped, try reading again.
+ if (lastRead < expectedLength)
+ {
+ // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
+ while (dest.remaining() != 0 && lastRead != -1)
+ {
+ lastRead = source.read(dest);
+
+ // if we got EOF, do not add to total read.
+ if (lastRead != -1)
+ {
+ totalRead += lastRead;
+ }
+ }
+ }
+
+ if (totalRead > 0)
+ {
+ dest.limit(dest.position());
+ }
+ else
+ {
+ dest.position(dest.limit());
+ }
+
+ return totalRead;
+ }
+
+ static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) throws IOException
+ {
+ if (skip <= 0) {
+ return 0;
+ }
+
+ int toSkip = skip;
+ int skipped = 0;
+ while(toSkip > 0 && skipped != -1) {
+ buffer.clear();
+ if (toSkip < buffer.capacity()) {
+ buffer.limit(toSkip);
+ }
+
+ skipped = source.read(buffer);
+ if (skipped > 0) {
+ toSkip -= skipped;
+ }
+ }
+
+ buffer.clear();
+ return skip - toSkip;
+ }
+
+ private static Class lookupClassQuietly(String name) {
+ try {
+ return SnappyFramed.class.getClassLoader().loadClass(name);
+ } catch (Throwable t) {
+ Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t);
+ }
+
+ return null;
+ }
+
+ /**
+ * Provides jvm implementation specific operation to aggressively release resources associated with buffer.
+ * @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}.
+ */
+ static void releaseDirectByteBuffer(ByteBuffer buffer)
+ {
+ assert buffer != null && buffer.isDirect();
+
+ if (SUN_DIRECT_BUFFER != null && SUN_DIRECT_BUFFER.isAssignableFrom(buffer.getClass())) {
+ try {
+ Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null);
+ SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null);
+ } catch (Throwable t) {
+ Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t);
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/xerial/snappy/SnappyNative.cpp b/src/main/java/org/xerial/snappy/SnappyNative.cpp
index 78af982..d635880 100755
--- a/src/main/java/org/xerial/snappy/SnappyNative.cpp
+++ b/src/main/java/org/xerial/snappy/SnappyNative.cpp
@@ -1,309 +1,309 @@
-/*--------------------------------------------------------------------------
- * Copyright 2011 Taro L. Saito
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *--------------------------------------------------------------------------*/
-#include - * DO NOT USE THIS CLASS since the direct use of this class might break the - * native library code loading in {@link SnappyLoader}. - *
- * - * @author leo - * - */ -public class SnappyNative -{ - - public native String nativeLibraryVersion(); - - // ------------------------------------------------------------------------ - // Generic compression/decompression routines. - // ------------------------------------------------------------------------ - public native long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException; - public native long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException; - - public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, - int outputOffset) throws IOException; - - public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException; - - public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, - int outputOffset) throws IOException; - - public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) - throws IOException; - - // Returns the maximal size of the compressed representation of - // input data that is "source_bytes" bytes in length; - public native int maxCompressedLength(int source_bytes); - - // This operation takes O(1) time. - public native int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException; - - public native int uncompressedLength(Object input, int offset, int len) throws IOException; - - public native long uncompressedLength(long inputAddr, long len) throws IOException; - - public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException; - - public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; - - public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException; - - public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; - - public void throw_error(int errorCode) throws IOException { - throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode)); - } - -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// snappy-java Project +// +// SnappyNative.java +// Since: 2011/03/30 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * JNI interface of the {@link Snappy} implementation. The native method in this class is + * defined in SnappyNative.h (genereted by javah) and SnappyNative.cpp + * + *+ * DO NOT USE THIS CLASS since the direct use of this class might break the + * native library code loading in {@link SnappyLoader}. + *
+ * + * @author leo + * + */ +public class SnappyNative +{ + + public native String nativeLibraryVersion(); + + // ------------------------------------------------------------------------ + // Generic compression/decompression routines. + // ------------------------------------------------------------------------ + public native long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException; + public native long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException; + + public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, + int outputOffset) throws IOException; + + public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException; + + public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, + int outputOffset) throws IOException; + + public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) + throws IOException; + + // Returns the maximal size of the compressed representation of + // input data that is "source_bytes" bytes in length; + public native int maxCompressedLength(int source_bytes); + + // This operation takes O(1) time. + public native int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException; + + public native int uncompressedLength(Object input, int offset, int len) throws IOException; + + public native long uncompressedLength(long inputAddr, long len) throws IOException; + + public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException; + + public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; + + public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException; + + public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; + + public void throw_error(int errorCode) throws IOException { + throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode)); + } + +} diff --git a/src/main/java/org/xerial/snappy/package-info.java b/src/main/java/org/xerial/snappy/package-info.java index 98f4d95..ab691e0 100755 --- a/src/main/java/org/xerial/snappy/package-info.java +++ b/src/main/java/org/xerial/snappy/package-info.java @@ -1,44 +1,44 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ - -/** - * Snappy API for compressing/decompressing data. - * - * Usage - * First, import {@link org.xerial.snappy.Snappy} in your Java code: - *
- *
- * import org.xerial.snappy.Snappy;
- *
- *
- * Then use {@link org.xerial.snappy.Snappy#compress(byte[])} and {@link org.xerial.snappy.Snappy#uncompress(byte[])}:
- *
- *
- * String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of Snappy, a fast compresser/decompresser.";
- * byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
- * byte[] uncompressed = Snappy.uncompress(compressed);
- * String result = new String(uncompressed, "UTF-8");
- * System.out.println(result);
- *
- *
- *
- * In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used.
- * - *
+ *
+ * import org.xerial.snappy.Snappy;
+ *
+ *
+ * Then use {@link org.xerial.snappy.Snappy#compress(byte[])} and {@link org.xerial.snappy.Snappy#uncompress(byte[])}:
+ *
+ *
+ * String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of Snappy, a fast compresser/decompresser.";
+ * byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
+ * byte[] uncompressed = Snappy.uncompress(compressed);
+ * String result = new String(uncompressed, "UTF-8");
+ * System.out.println(result);
+ *
+ *
+ *
+ * In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used.
+ * + *- * This has been copied from dain's snappy implementation.. - *
- */ -public class RandomGenerator { - - public final byte[] data; - public int position; - - public RandomGenerator(double compressionRatio) { - // We use a limited amount of data over and over again and ensure - // that it is larger than the compression window (32KB), and also - // large enough to serve all typical value sizes we want to write. - Random rnd = new Random(301); - data = new byte[1048576 + 100]; - for (int i = 0; i < 1048576; i += 100) { - // Add a short fragment that is as compressible as specified ratio - System.arraycopy(compressibleData(rnd, compressionRatio, 100), 0, - data, i, 100); - } - } - - public int getNextPosition(int length) { - if (position + length > data.length) { - position = 0; - assert (length < data.length); - } - int result = position; - position += length; - return result; - } - - private static byte[] compressibleData(Random random, - double compressionRatio, int length) { - int raw = (int) (length * compressionRatio); - if (raw < 1) { - raw = 1; - } - byte[] rawData = generateRandomData(random, raw); - - // Duplicate the random data until we have filled "length" bytes - byte[] dest = new byte[length]; - for (int i = 0; i < length;) { - int chunkLength = Math.min(rawData.length, length - i); - System.arraycopy(rawData, 0, dest, i, chunkLength); - i += chunkLength; - } - return dest; - } - - private static byte[] generateRandomData(Random random, int length) { - byte[] rawData = new byte[length]; - for (int i = 0; i < rawData.length; i++) { - rawData[i] = (byte) random.nextInt(256); - } - return rawData; - } +/* + * Created: Apr 15, 2013 + */ +package org.xerial.snappy; + +import java.util.Random; + +/** + * Generates random data with specific expected snappy performance characteristics. + * + *+ * This has been copied from dain's snappy implementation.. + *
+ */ +public class RandomGenerator { + + public final byte[] data; + public int position; + + public RandomGenerator(double compressionRatio) { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + // large enough to serve all typical value sizes we want to write. + Random rnd = new Random(301); + data = new byte[1048576 + 100]; + for (int i = 0; i < 1048576; i += 100) { + // Add a short fragment that is as compressible as specified ratio + System.arraycopy(compressibleData(rnd, compressionRatio, 100), 0, + data, i, 100); + } + } + + public int getNextPosition(int length) { + if (position + length > data.length) { + position = 0; + assert (length < data.length); + } + int result = position; + position += length; + return result; + } + + private static byte[] compressibleData(Random random, + double compressionRatio, int length) { + int raw = (int) (length * compressionRatio); + if (raw < 1) { + raw = 1; + } + byte[] rawData = generateRandomData(random, raw); + + // Duplicate the random data until we have filled "length" bytes + byte[] dest = new byte[length]; + for (int i = 0; i < length;) { + int chunkLength = Math.min(rawData.length, length - i); + System.arraycopy(rawData, 0, dest, i, chunkLength); + i += chunkLength; + } + return dest; + } + + private static byte[] generateRandomData(Random random, int length) { + byte[] rawData = new byte[length]; + for (int i = 0; i < rawData.length; i++) { + rawData[i] = (byte) random.nextInt(256); + } + return rawData; + } } \ No newline at end of file diff --git a/src/test/java/org/xerial/snappy/SnappyInputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyInputStreamTest.java index bb5f26e..c1c9930 100755 --- a/src/test/java/org/xerial/snappy/SnappyInputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyInputStreamTest.java @@ -1,179 +1,179 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// XerialJ -// -// SnappyInputStreamTest.java -// Since: 2011/03/31 22:31:51 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -import static org.junit.Assert.*; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.junit.Test; -import org.xerial.util.FileResource; -import org.xerial.util.log.Logger; -import scala.Array; - -public class SnappyInputStreamTest -{ - private static Logger _logger = Logger.getLogger(SnappyInputStreamTest.class); - - public static byte[] readResourceFile(String fileName) throws IOException { - BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class, fileName) - .openStream()); - assertNotNull(input); - return readFully(input); - } - - public static byte[] readFully(InputStream input) throws IOException { - try { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buf = new byte[4096]; - for (int readBytes = 0; (readBytes = input.read(buf)) != -1;) { - out.write(buf, 0, readBytes); - } - out.flush(); - return out.toByteArray(); - } - finally { - input.close(); - } - } - - public static byte[] byteWiseReadFully(InputStream input) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buf = new byte[4096]; - for (int readData = 0; (readData = input.read()) != -1;) { - out.write(readData); - } - out.flush(); - return out.toByteArray(); - } - - @Test - public void read() throws Exception { - ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); - SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); - byte[] orig = readResourceFile("alice29.txt"); - snappyOut.write(orig); - snappyOut.close(); - byte[] compressed = compressedBuf.toByteArray(); - _logger.debug("compressed size: " + compressed.length); - - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); - byte[] uncompressed = readFully(in); - - assertEquals(orig.length, uncompressed.length); - assertArrayEquals(orig, uncompressed); - - } - - @Test - public void readBlockCompressedData() throws Exception { - byte[] orig = readResourceFile("alice29.txt"); - byte[] compressed = Snappy.compress(orig); - - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); - byte[] uncompressed = readFully(in); - - assertEquals(orig.length, uncompressed.length); - assertArrayEquals(orig, uncompressed); - } - - @Test - public void biteWiseRead() throws Exception { - byte[] orig = readResourceFile("testdata/calgary/paper6"); - byte[] compressed = Snappy.compress(orig); - - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); - byte[] uncompressed = byteWiseReadFully(in); - - assertEquals(orig.length, uncompressed.length); - assertArrayEquals(orig, uncompressed); - - } - - @Test - public void available() throws Exception { - byte[] orig = readResourceFile("testdata/calgary/paper6"); - byte[] compressed = Snappy.compress(orig); - - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); - byte[] buf = new byte[4]; - for (int readBytes = 0; (readBytes = in.read(buf)) != -1;) { - assertTrue(in.available() >= 0); - } - assertTrue(in.available() == 0); - in.close(); - } - - @Test - public void emptyStream() throws Exception { - try { - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(new byte[0])); - byte[] uncompressed = readFully(in); - assertEquals(0, uncompressed.length); - fail("should not reach here"); - } - catch(SnappyIOException e) { - assertEquals(SnappyErrorCode.EMPTY_INPUT, e.getErrorCode()); - } - } - - public static byte[] compressResource(String resourcePath) throws Exception { - ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); - SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); - byte[] orig = readResourceFile(resourcePath); - snappyOut.write(orig); - snappyOut.close(); - return compressedBuf.toByteArray(); - } - - @Test - public void chunkRead() throws Exception { - byte[] chunk1 = compressResource("alice29.txt"); - byte[] chunk2 = compressResource("testdata/calgary/paper6"); - - byte[] concatenated = new byte[chunk1.length + chunk2.length]; - System.arraycopy(chunk1, 0, concatenated, 0, chunk1.length); - System.arraycopy(chunk2, 0, concatenated, chunk1.length, chunk2.length); - - SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(concatenated)); - byte[] uncompressed = readFully(in); - - byte[] orig1 = readResourceFile("alice29.txt"); - byte[] orig2 = readResourceFile("testdata/calgary/paper6"); - assertEquals(orig1.length + orig2.length, uncompressed.length); - byte[] uncompressed1 = new byte[orig1.length]; - byte[] uncompressed2 = new byte[orig2.length]; - System.arraycopy(uncompressed, 0, uncompressed1, 0, orig1.length); - System.arraycopy(uncompressed, orig1.length, uncompressed2, 0, orig2.length); - - assertArrayEquals(orig1, uncompressed1); - assertArrayEquals(orig2, uncompressed2); - } - -} +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyInputStreamTest.java +// Since: 2011/03/31 22:31:51 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import static org.junit.Assert.*; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.junit.Test; +import org.xerial.util.FileResource; +import org.xerial.util.log.Logger; +import scala.Array; + +public class SnappyInputStreamTest +{ + private static Logger _logger = Logger.getLogger(SnappyInputStreamTest.class); + + public static byte[] readResourceFile(String fileName) throws IOException { + BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class, fileName) + .openStream()); + assertNotNull(input); + return readFully(input); + } + + public static byte[] readFully(InputStream input) throws IOException { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + for (int readBytes = 0; (readBytes = input.read(buf)) != -1;) { + out.write(buf, 0, readBytes); + } + out.flush(); + return out.toByteArray(); + } + finally { + input.close(); + } + } + + public static byte[] byteWiseReadFully(InputStream input) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + for (int readData = 0; (readData = input.read()) != -1;) { + out.write(readData); + } + out.flush(); + return out.toByteArray(); + } + + @Test + public void read() throws Exception { + ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); + SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); + byte[] orig = readResourceFile("alice29.txt"); + snappyOut.write(orig); + snappyOut.close(); + byte[] compressed = compressedBuf.toByteArray(); + _logger.debug("compressed size: " + compressed.length); + + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); + byte[] uncompressed = readFully(in); + + assertEquals(orig.length, uncompressed.length); + assertArrayEquals(orig, uncompressed); + + } + + @Test + public void readBlockCompressedData() throws Exception { + byte[] orig = readResourceFile("alice29.txt"); + byte[] compressed = Snappy.compress(orig); + + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); + byte[] uncompressed = readFully(in); + + assertEquals(orig.length, uncompressed.length); + assertArrayEquals(orig, uncompressed); + } + + @Test + public void biteWiseRead() throws Exception { + byte[] orig = readResourceFile("testdata/calgary/paper6"); + byte[] compressed = Snappy.compress(orig); + + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); + byte[] uncompressed = byteWiseReadFully(in); + + assertEquals(orig.length, uncompressed.length); + assertArrayEquals(orig, uncompressed); + + } + + @Test + public void available() throws Exception { + byte[] orig = readResourceFile("testdata/calgary/paper6"); + byte[] compressed = Snappy.compress(orig); + + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); + byte[] buf = new byte[4]; + for (int readBytes = 0; (readBytes = in.read(buf)) != -1;) { + assertTrue(in.available() >= 0); + } + assertTrue(in.available() == 0); + in.close(); + } + + @Test + public void emptyStream() throws Exception { + try { + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(new byte[0])); + byte[] uncompressed = readFully(in); + assertEquals(0, uncompressed.length); + fail("should not reach here"); + } + catch(SnappyIOException e) { + assertEquals(SnappyErrorCode.EMPTY_INPUT, e.getErrorCode()); + } + } + + public static byte[] compressResource(String resourcePath) throws Exception { + ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); + SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); + byte[] orig = readResourceFile(resourcePath); + snappyOut.write(orig); + snappyOut.close(); + return compressedBuf.toByteArray(); + } + + @Test + public void chunkRead() throws Exception { + byte[] chunk1 = compressResource("alice29.txt"); + byte[] chunk2 = compressResource("testdata/calgary/paper6"); + + byte[] concatenated = new byte[chunk1.length + chunk2.length]; + System.arraycopy(chunk1, 0, concatenated, 0, chunk1.length); + System.arraycopy(chunk2, 0, concatenated, chunk1.length, chunk2.length); + + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(concatenated)); + byte[] uncompressed = readFully(in); + + byte[] orig1 = readResourceFile("alice29.txt"); + byte[] orig2 = readResourceFile("testdata/calgary/paper6"); + assertEquals(orig1.length + orig2.length, uncompressed.length); + byte[] uncompressed1 = new byte[orig1.length]; + byte[] uncompressed2 = new byte[orig2.length]; + System.arraycopy(uncompressed, 0, uncompressed1, 0, orig1.length); + System.arraycopy(uncompressed, orig1.length, uncompressed2, 0, orig2.length); + + assertArrayEquals(orig1, uncompressed1); + assertArrayEquals(orig2, uncompressed2); + } + +} diff --git a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java index 9bcc85f..a27197f 100755 --- a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java +++ b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java @@ -1,125 +1,125 @@ -/*-------------------------------------------------------------------------- - * Copyright 2011 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ -//-------------------------------------- -// XerialJ -// -// SnappyLoaderTest.java -// Since: 2011/06/22 23:59:47 -// -// $URL$ -// $Author$ -//-------------------------------------- -package org.xerial.snappy; - -import org.codehaus.plexus.classworlds.ClassWorld; -import org.codehaus.plexus.classworlds.realm.ClassRealm; -import org.junit.Test; -import org.xerial.util.FileResource; -import org.xerial.util.log.Logger; - -import java.io.*; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.fail; - -public class SnappyLoaderTest -{ - private static Logger _logger = Logger.getLogger(SnappyLoaderTest.class); - - public static BufferedInputStream openByteStream(Class< ? > referenceClass, String resourceFileName) - throws IOException { - URL url = FileResource.find(referenceClass, resourceFileName); - if (url != null) { - return new BufferedInputStream(url.openStream()); - } - else - return null; - } - - public static