Fixes issue 4

This commit is contained in:
Taro L. Saito 2011-03-31 17:06:33 +09:00
parent 4935319db1
commit 41debf9cb6
6 changed files with 175 additions and 41 deletions

View File

@ -45,12 +45,12 @@ import java.util.Properties;
*/ */
public class LoadSnappy public class LoadSnappy
{ {
private static boolean extracted = false; private static boolean isLoaded = false;
public static boolean initialize() { public static boolean load() {
if (!extracted) if (!isLoaded)
loadSnappyNativeLibrary(); loadSnappyNativeLibrary();
return extracted; return isLoaded;
} }
/** /**
@ -164,7 +164,7 @@ public class LoadSnappy
} }
private static void loadSnappyNativeLibrary() { private static void loadSnappyNativeLibrary() {
if (extracted) if (isLoaded)
return; return;
// Try loading library from org.sqlite.lib.path library path */ // Try loading library from org.sqlite.lib.path library path */
@ -177,7 +177,7 @@ public class LoadSnappy
if (snappyNativeLibraryPath != null) { if (snappyNativeLibraryPath != null) {
if (loadNativeLibrary(snappyNativeLibraryPath, snappyNativeLibraryName)) { if (loadNativeLibrary(snappyNativeLibraryPath, snappyNativeLibraryName)) {
extracted = true; isLoaded = true;
return; return;
} }
} }
@ -194,11 +194,11 @@ public class LoadSnappy
String tempFolder = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); String tempFolder = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath();
// Try extracting the library from jar // Try extracting the library from jar
if (extractAndLoadLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder)) { if (extractAndLoadLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder)) {
extracted = true; isLoaded = true;
return; return;
} }
extracted = false; isLoaded = false;
return; return;
} }

View File

@ -27,7 +27,7 @@ package org.xerial.snappy;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/** /**
* Snappy API * Snappy API for data compression/decompression
* *
* @author leo * @author leo
* *
@ -35,6 +35,11 @@ import java.nio.ByteBuffer;
public class Snappy public class Snappy
{ {
/**
* Get the native library version of the snappy
*
* @return native library version
*/
public static String getNativeLibraryVersion() { public static String getNativeLibraryVersion() {
return SnappyNative.nativeLibraryVersion(); return SnappyNative.nativeLibraryVersion();
} }
@ -53,7 +58,7 @@ public class Snappy
* @throws SnappyError * @throws SnappyError
* when the input is not a direct buffer * when the input is not a direct buffer
*/ */
public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) { public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) throws SnappyException {
if (!uncompressed.isDirect()) if (!uncompressed.isDirect())
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
@ -73,9 +78,37 @@ public class Snappy
return compressedSize; return compressedSize;
} }
/**
* Compress the input buffer content in [inputOffset,
* ...inputOffset+inputLength) then output to the specified output buffer.
*
* @param input
* @param inputOffset
* @param inputLength
* @param output
* @param outputOffset
* @return byte size of the compressed data
* @throws SnappyException
* when failed to access the input/output buffer
*/
public static int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws SnappyException {
if (input == null || output == null)
throw new NullPointerException("input or output is null");
int compressedSize = SnappyNative.rawCompress(input, inputOffset, inputLength, output, outputOffset);
return compressedSize;
}
/** /**
* Uncompress the content in the input buffer. The result is dumped to the * Uncompress the content in the input buffer. The result is dumped to the
* specified output buffer * specified output buffer.
*
* Note that if you pass the wrong data or the range [pos(), limit()) that
* cannot be uncompressed, your JVM might crash due to the access violation
* exception issued in the native code written in C++. To avoid this type of
* crash, use {@link #isValidCompressedBuffer(ByteBuffer)} first.
*
* *
* @param compressed * @param compressed
* buffer[pos() ... limit()) containing the input data * buffer[pos() ... limit()) containing the input data
@ -107,6 +140,32 @@ public class Snappy
return decompressedSize; return decompressedSize;
} }
/**
* Uncompress the content in the input buffer. The uncompressed data is
* written to the output buffer.
*
* Note that if you pass the wrong data or the range [inputOffset,
* inputOffset + inputLength) that cannot be uncompressed, your JVM might
* crash due to the access violation exception issued in the native code
* written in C++. To avoid this type of crash, use
* {@link #isValidCompressedBuffer(byte[], int, int)} first.
*
* @param input
* @param inputOffset
* @param inputLength
* @param output
* @param outputOffset
* @return
* @throws SnappyException
*/
public static int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws SnappyException {
if (input == null || output == null)
throw new NullPointerException("input or output is null");
return SnappyNative.rawUncompress(input, inputOffset, inputLength, output, outputOffset);
}
/** /**
* Get the uncompressed byte size of the given compressed input. * Get the uncompressed byte size of the given compressed input.
* *
@ -125,6 +184,21 @@ public class Snappy
return SnappyNative.uncompressedLength(compressed, compressed.position(), compressed.remaining()); return SnappyNative.uncompressedLength(compressed, compressed.position(), compressed.remaining());
} }
/**
* Get the uncompressed byte size of the given compressed input
*
* @param input
* @param offset
* @param length
* @return umcompressed byte size of the the given input data
* @throws SnappyException
*/
public static int uncompressedLength(byte[] input, int offset, int length) throws SnappyException {
if (input == null)
throw new NullPointerException("input is null");
return SnappyNative.uncompressedLength(input, offset, length);
}
/** /**
* Get the maximum byte size needed for compressing a data of the given byte * Get the maximum byte size needed for compressing a data of the given byte
* size. * size.
@ -140,11 +214,23 @@ public class Snappy
/** /**
* Returns true iff the contents of compressed buffer [pos() ... limit()) * Returns true iff the contents of compressed buffer [pos() ... limit())
* can be uncompressed successfully. Does not return the uncompressed data. * can be uncompressed successfully. Does not return the uncompressed data.
* Takes time proportional to compressed_length, but is usually at least a * Takes time proportional to the input length, but is usually at least a
* factor of four faster than actual decompression. * factor of four faster than actual decompression.
*/ */
public static boolean isValidCompressedBuffer(ByteBuffer compressed) { public static boolean isValidCompressedBuffer(ByteBuffer compressed) throws SnappyException {
return SnappyNative.isValidCompressedBuffer(compressed, compressed.position(), compressed.remaining()); return SnappyNative.isValidCompressedBuffer(compressed, compressed.position(), compressed.remaining());
} }
/**
* Returns true iff the contents of compressed buffer [offset,
* offset+length) can be uncompressed successfully. Does not return the
* uncompressed data. Takes time proportional to the input length, but is
* usually at least a factor of four faster than actual decompression.
*/
public static boolean isValidCompressedBuffer(byte[] input, int offset, int length) throws SnappyException {
if (input == null)
throw new NullPointerException("input is null");
return SnappyNative.isValidCompressedBuffer(input, offset, length);
}
} }

View File

@ -42,11 +42,15 @@ JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersi
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv* env, jclass self, jobject uncompressed, jint upos, jint ulen, jobject compressed, jint cpos) (JNIEnv* env, jclass self, jobject uncompressed, jint upos, jint ulen, jobject compressed, jint cpos)
{ {
char* uncompressedBuffer = (char*) env->GetDirectBufferAddress(uncompressed) + upos; char* uncompressedBuffer = (char*) env->GetDirectBufferAddress(uncompressed);
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed) + cpos; char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
size_t compressedLength; if(uncompressedBuffer == 0 || compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
snappy::RawCompress(uncompressedBuffer, (size_t) ulen, compressedBuffer, &compressedLength); size_t compressedLength;
snappy::RawCompress(uncompressedBuffer + upos, (size_t) ulen, compressedBuffer + cpos, &compressedLength);
return (jint) compressedLength; return (jint) compressedLength;
} }
@ -85,12 +89,16 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress___3BII_3B
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen, jobject decompressed, jint dpos) (JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen, jobject decompressed, jint dpos)
{ {
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed) + cpos; char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
char* decompressedBuffer = (char*) env->GetDirectBufferAddress(decompressed) + dpos; char* decompressedBuffer = (char*) env->GetDirectBufferAddress(decompressed);
if(compressedBuffer == 0 || decompressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t decompressedLength; size_t decompressedLength;
snappy::GetUncompressedLength(compressedBuffer, (size_t) clen, &decompressedLength); snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &decompressedLength);
bool ret = snappy::RawUncompress(compressedBuffer, (size_t) clen, decompressedBuffer); bool ret = snappy::RawUncompress(compressedBuffer + cpos, (size_t) clen, decompressedBuffer + dpos);
if(!ret) { if(!ret) {
throw_exception(env, self, 2); throw_exception(env, self, 2);
return 0; return 0;
@ -147,9 +155,14 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_maxCompressedLength
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_nio_ByteBuffer_2II JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen) (JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen)
{ {
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed) + cpos; char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t result; size_t result;
bool ret = snappy::GetUncompressedLength(compressedBuffer, (size_t) clen, &result); bool ret = snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &result);
if(!ret) { if(!ret) {
throw_exception(env, self, 2); throw_exception(env, self, 2);
return 0; return 0;
@ -177,8 +190,12 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength___
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen) (JNIEnv * env, jclass self, jobject compressed, jint cpos, jint clen)
{ {
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed) + cpos; char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
bool ret = snappy::IsValidCompressedBuffer(compressedBuffer, (size_t) clen); if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
bool ret = snappy::IsValidCompressedBuffer(compressedBuffer + cpos, (size_t) clen);
return ret; return ret;
} }

View File

@ -35,7 +35,7 @@ import java.nio.ByteBuffer;
public class SnappyNative public class SnappyNative
{ {
static { static {
LoadSnappy.initialize(); LoadSnappy.load();
} }
public native static String nativeLibraryVersion(); public native static String nativeLibraryVersion();
@ -44,7 +44,7 @@ public class SnappyNative
// Generic compression/decompression routines. // Generic compression/decompression routines.
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
public native static int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, public native static int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed,
int outputOffset); int outputOffset) throws SnappyException;
public native static int rawCompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) public native static int rawCompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws SnappyException; throws SnappyException;
@ -64,9 +64,10 @@ public class SnappyNative
public native static int uncompressedLength(byte[] input, int offset, int len) throws SnappyException; public native static int uncompressedLength(byte[] input, int offset, int len) throws SnappyException;
public native static boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len); public native static boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len)
throws SnappyException;
public native static boolean isValidCompressedBuffer(byte[] input, int offset, int len); public native static boolean isValidCompressedBuffer(byte[] input, int offset, int len) throws SnappyException;
public static void throw_error(int errorCode) throws SnappyException { public static void throw_error(int errorCode) throws SnappyException {
throw new SnappyException(errorCode); throw new SnappyException(errorCode);

View File

@ -39,7 +39,7 @@ public class SnappyTest
@Test @Test
public void getVersion() throws Exception { public void getVersion() throws Exception {
String version = Snappy.getNativeLibraryVersion(); String version = Snappy.getNativeLibraryVersion();
_logger.info("version: " + version); _logger.debug("version: " + version);
} }
@Test @Test
@ -62,7 +62,7 @@ public class SnappyTest
} }
@Test @Test
public void load() throws Exception { public void directBuffer() throws Exception {
StringBuilder s = new StringBuilder(); StringBuilder s = new StringBuilder();
for (int i = 0; i < 20; ++i) { for (int i = 0; i < 20; ++i) {
@ -74,13 +74,13 @@ public class SnappyTest
ByteBuffer src = ByteBuffer.allocateDirect(orig.length); ByteBuffer src = ByteBuffer.allocateDirect(orig.length);
src.put(orig); src.put(orig);
src.flip(); src.flip();
_logger.info("input size: " + src.remaining()); _logger.debug("input size: " + src.remaining());
int maxCompressedLen = Snappy.maxCompressedLength(src.remaining()); int maxCompressedLen = Snappy.maxCompressedLength(src.remaining());
_logger.info("max compressed length:" + maxCompressedLen); _logger.debug("max compressed length:" + maxCompressedLen);
ByteBuffer compressed = ByteBuffer.allocateDirect(maxCompressedLen); ByteBuffer compressed = ByteBuffer.allocateDirect(maxCompressedLen);
int compressedSize = Snappy.compress(src, compressed); int compressedSize = Snappy.compress(src, compressed);
_logger.info("compressed length: " + compressedSize); _logger.debug("compressed length: " + compressedSize);
assertTrue(Snappy.isValidCompressedBuffer(compressed)); assertTrue(Snappy.isValidCompressedBuffer(compressed));
@ -93,7 +93,7 @@ public class SnappyTest
assertEquals(compressedSize, compressed.remaining()); assertEquals(compressedSize, compressed.remaining());
int uncompressedLen = Snappy.uncompressedLength(compressed); int uncompressedLen = Snappy.uncompressedLength(compressed);
_logger.info("uncompressed length: " + uncompressedLen); _logger.debug("uncompressed length: " + uncompressedLen);
ByteBuffer extract = ByteBuffer.allocateDirect(uncompressedLen); ByteBuffer extract = ByteBuffer.allocateDirect(uncompressedLen);
int uncompressedLen2 = Snappy.uncompress(compressed, extract); int uncompressedLen2 = Snappy.uncompress(compressed, extract);
assertEquals(uncompressedLen, uncompressedLen2); assertEquals(uncompressedLen, uncompressedLen2);
@ -102,13 +102,13 @@ public class SnappyTest
byte[] b = new byte[uncompressedLen]; byte[] b = new byte[uncompressedLen];
extract.get(b); extract.get(b);
String decompressed = new String(b); String decompressed = new String(b);
_logger.info(decompressed); _logger.debug(decompressed);
assertEquals(origStr, decompressed); assertEquals(origStr, decompressed);
} }
@Test @Test
public void intermediateBuffer() throws Exception { public void bufferOffset() throws Exception {
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA"; String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] orig = m.getBytes(); byte[] orig = m.getBytes();
@ -146,19 +146,49 @@ public class SnappyTest
} }
@Test @Test
public void rawCompress() throws Exception { public void byteArrayCompress() throws Exception {
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA"; String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] input = m.getBytes(); byte[] input = m.getBytes();
byte[] output = new byte[Snappy.maxCompressedLength(input.length)]; byte[] output = new byte[Snappy.maxCompressedLength(input.length)];
int compressedSize = SnappyNative.rawCompress(input, 0, input.length, output, 0); int compressedSize = Snappy.compress(input, 0, input.length, output, 0);
byte[] uncompressed = new byte[input.length]; byte[] uncompressed = new byte[input.length];
assertTrue(SnappyNative.isValidCompressedBuffer(output, 0, compressedSize)); assertTrue(Snappy.isValidCompressedBuffer(output, 0, compressedSize));
int uncompressedSize = SnappyNative.rawUncompress(output, 0, compressedSize, uncompressed, 0); int uncompressedSize = Snappy.uncompress(output, 0, compressedSize, uncompressed, 0);
String m2 = new String(uncompressed); String m2 = new String(uncompressed);
assertEquals(m, m2); assertEquals(m, m2);
} }
@Test
public void rangeCheck() throws Exception {
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] input = m.getBytes();
byte[] output = new byte[Snappy.maxCompressedLength(input.length)];
int compressedSize = Snappy.compress(input, 0, input.length, output, 0);
assertTrue(Snappy.isValidCompressedBuffer(output, 0, compressedSize));
// Intentionally set an invalid range
assertFalse(Snappy.isValidCompressedBuffer(output, 0, compressedSize + 1));
assertFalse(Snappy.isValidCompressedBuffer(output, 1, compressedSize));
// Test the ByteBuffer API
ByteBuffer bin = ByteBuffer.allocateDirect(input.length);
bin.put(input);
bin.flip();
ByteBuffer bout = ByteBuffer.allocateDirect(Snappy.maxCompressedLength(bin.remaining()));
int compressedSize2 = Snappy.compress(bin, bout);
assertEquals(compressedSize, compressedSize2);
assertTrue(Snappy.isValidCompressedBuffer(bout));
// Intentionally set an invalid range
bout.limit(bout.limit() + 1);
assertFalse(Snappy.isValidCompressedBuffer(bout));
bout.limit(bout.limit() - 1);
bout.position(1);
assertFalse(Snappy.isValidCompressedBuffer(bout));
}
} }