From 23f528095cde6fbd2baae9390dfc7eaaf2eaf92e Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Wed, 25 Jan 2017 23:26:23 +0900 Subject: [PATCH] Implement ByteBuffer-based APIs for BitShuffle (#155) --- .../java/org/xerial/snappy/BitShuffle.java | 81 ++++++ .../org/xerial/snappy/BitShuffleNative.cpp | 46 +++- .../java/org/xerial/snappy/BitShuffleNative.h | 16 ++ .../org/xerial/snappy/BitShuffleNative.java | 7 + .../org/xerial/snappy/BitShuffleType.java | 53 ++++ .../org/xerial/snappy/BitShuffleTest.java | 238 ++++++++++++++++++ 6 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/xerial/snappy/BitShuffleType.java diff --git a/src/main/java/org/xerial/snappy/BitShuffle.java b/src/main/java/org/xerial/snappy/BitShuffle.java index 7cca5be..5c14c73 100644 --- a/src/main/java/org/xerial/snappy/BitShuffle.java +++ b/src/main/java/org/xerial/snappy/BitShuffle.java @@ -25,6 +25,7 @@ package org.xerial.snappy; import java.io.IOException; +import java.nio.ByteBuffer; public class BitShuffle { @@ -42,6 +43,46 @@ public class BitShuffle */ private static BitShuffleNative impl; + /** + * Apply a bit-shuffling filter into the content in the given input buffer. After bit-shuffling, + * you can retrieve the shuffled data from the output buffer [pos() ...limit()) + * (shuffled data size = limit() - pos() = remaining()). + * + * @param input buffer[pos() ... limit()) containing the input data + * @param type element type of the input data + * @param shuffled output of the shuffled data. Uses range [pos()..]. + * @return byte size of the shuffled data. + * @throws SnappyError when the input is not a direct buffer + * @throws IllegalArgumentException when the input length is not a multiple of a given type size + */ + public static int bitShuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuffled) throws IOException { + if (!input.isDirect()) { + throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); + } + if (!shuffled.isDirect()) { + throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer"); + } + + // input: input[pos(), limit()) + // output: shuffled + int uPos = input.position(); + int uLen = input.remaining(); + int typeSize = type.getTypeSize(); + if (uLen % typeSize != 0) { + throw new IllegalArgumentException("input length must be a multiple of a given type size"); + } + if (shuffled.remaining() < uLen) { + throw new IllegalArgumentException("not enough space for output"); + } + int numProcessed = impl.bitShuffleInDirectBuffer(input, uPos, typeSize, uLen, shuffled, shuffled.position()); + assert(numProcessed == uLen); + + // pos limit + // [ ......BBBBBBB.........] + shuffled.limit(shuffled.position() + numProcessed); + return numProcessed; + } + /** * Apply a bit-shuffling filter into the input short array. * @@ -112,6 +153,46 @@ public class BitShuffle return output; } + /** + * Convert the input bit-shuffled byte array into an original array. The result is dumped + * to the specified output buffer. + * + * @param shuffled buffer[pos() ... limit()) containing the input shuffled data + * @param type element type of the input data + * @param output output of the the original data. It uses buffer[pos()..] + * @return byte size of the unshuffled data. + * @throws IOException when failed to unshuffle the given input + * @throws SnappyError when the input is not a direct buffer + * @throws IllegalArgumentException when the length of input shuffled data is not a multiple of a given type size + */ + public static int bitUnShuffle(ByteBuffer shuffled, BitShuffleType type, ByteBuffer output) throws IOException { + if (!shuffled.isDirect()) { + throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); + } + if (!output.isDirect()) { + throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer"); + } + + // input: input[pos(), limit()) + // output: shuffled + int uPos = shuffled.position(); + int uLen = shuffled.remaining(); + int typeSize = type.getTypeSize(); + if (uLen % typeSize != 0) { + throw new IllegalArgumentException("length of input shuffled data must be a multiple of a given type size"); + } + if (output.remaining() < uLen) { + throw new IllegalArgumentException("not enough space for output"); + } + int numProcessed = impl.bitUnShuffleInDirectBuffer(shuffled, uPos, typeSize, uLen, output, shuffled.position()); + assert(numProcessed == uLen); + + // pos limit + // [ ......BBBBBBB.........] + shuffled.limit(shuffled.position() + numProcessed); + return numProcessed; + } + /** * Convert the input bit-shuffled byte array into an original short array. * diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.cpp b/src/main/java/org/xerial/snappy/BitShuffleNative.cpp index eb79a16..db767c6 100755 --- a/src/main/java/org/xerial/snappy/BitShuffleNative.cpp +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.cpp @@ -28,7 +28,7 @@ inline void throw_exception(JNIEnv *env, jobject self, int errorCode) } /* - * Class: org_xerial_snappy_SnappyNative + * Class: org_xerial_snappy_BitShuffleNative * Method: bitShuffle * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I */ @@ -59,7 +59,28 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle } /* - * Class: org_xerial_snappy_SnappyNative + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitShuffleInDirectBuffer + * Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer + (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset) +{ + char* inputBuffer = (char*) env->GetDirectBufferAddress(input); + char* outputBuffer = (char*) env->GetDirectBufferAddress(output); + if(inputBuffer == 0 || outputBuffer == 0) { + throw_exception(env, self, 3); + return (jint) 0; + } + + int64_t processedBytes = bshuf_bitshuffle( + inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0); + + return (jint) processedBytes; +} + +/* + * Class: org_xerial_snappy_BitShuffleNative * Method: bitUnShuffle * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I */ @@ -89,3 +110,24 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle return (jint) processedBytes; } +/* + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitUnShuffleInDirectBuffer + * Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer + (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset) +{ + char* inputBuffer = (char*) env->GetDirectBufferAddress(input); + char* outputBuffer = (char*) env->GetDirectBufferAddress(output); + if(inputBuffer == 0 || outputBuffer == 0) { + throw_exception(env, self, 3); + return (jint) 0; + } + + int64_t processedBytes = bshuf_bitunshuffle( + inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0); + + return (jint) processedBytes; +} + diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.h b/src/main/java/org/xerial/snappy/BitShuffleNative.h index 3f71339..9b5e335 100644 --- a/src/main/java/org/xerial/snappy/BitShuffleNative.h +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.h @@ -15,6 +15,14 @@ extern "C" { JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); +/* + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitShuffleInDirectBuffer + * Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer + (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); + /* * Class: org_xerial_snappy_BitShuffleNative * Method: bitUnShuffle @@ -23,6 +31,14 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); +/* + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitUnShuffleInDirectBuffer + * Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer + (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); + #ifdef __cplusplus } #endif diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.java b/src/main/java/org/xerial/snappy/BitShuffleNative.java index 57a61d4..bd63242 100644 --- a/src/main/java/org/xerial/snappy/BitShuffleNative.java +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.java @@ -25,6 +25,7 @@ package org.xerial.snappy; import java.io.IOException; +import java.nio.ByteBuffer; /** * JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is @@ -48,6 +49,12 @@ public class BitShuffleNative public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) throws IOException; + public native int bitShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset) + throws IOException; + public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) throws IOException; + + public native int bitUnShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset) + throws IOException; } diff --git a/src/main/java/org/xerial/snappy/BitShuffleType.java b/src/main/java/org/xerial/snappy/BitShuffleType.java new file mode 100644 index 0000000..d8c813a --- /dev/null +++ b/src/main/java/org/xerial/snappy/BitShuffleType.java @@ -0,0 +1,53 @@ +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// XerialJ +// +// SnappyErrorCode.java +// Since: 2011/03/30 14:56:50 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +/** + * Type codes used in ByteBuffer based BitShuffle APIs + * + * @author leo + */ +public enum BitShuffleType +{ + + BYTE(1), + SHORT(2), + INT(4), + LONG(8), + FLOAT(4), + DOUBLE(8); + + public final int id; + + private BitShuffleType(int id) + { + this.id = id; + } + + public int getTypeSize() + { + return id; + } +} diff --git a/src/test/java/org/xerial/snappy/BitShuffleTest.java b/src/test/java/org/xerial/snappy/BitShuffleTest.java index 49f904d..ab2bf3e 100644 --- a/src/test/java/org/xerial/snappy/BitShuffleTest.java +++ b/src/test/java/org/xerial/snappy/BitShuffleTest.java @@ -26,10 +26,248 @@ package org.xerial.snappy; import static org.junit.Assert.*; +import java.nio.ByteBuffer; + +import org.junit.Assert; import org.junit.Test; public class BitShuffleTest { + @Test + public void directBufferCheck() + throws Exception + { + ByteBuffer heapBuf = ByteBuffer.allocate(64); + ByteBuffer directBuf = ByteBuffer.allocateDirect(64); + + // Tests for BitShuffle.bitShuffle() + try { + BitShuffle.bitShuffle(heapBuf, BitShuffleType.BYTE, directBuf); + fail("no expected exception happened"); + } + catch (SnappyError e) { + Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER); + Assert.assertTrue(e.getMessage().contains("input is not a direct buffer")); + } + try { + BitShuffle.bitShuffle(directBuf, BitShuffleType.BYTE, heapBuf); + fail("no expected exception happened"); + } + catch (SnappyError e) { + Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER); + Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer")); + } + + // Then, tests for BitShuffle.bitUnShuffle() + try { + BitShuffle.bitUnShuffle(heapBuf, BitShuffleType.BYTE, directBuf); + fail("no expected exception happened"); + } + catch (SnappyError e) { + Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER); + Assert.assertTrue(e.getMessage().contains("input is not a direct buffer")); + } + try { + BitShuffle.bitUnShuffle(directBuf, BitShuffleType.BYTE, heapBuf); + fail("no expected exception happened"); + } + catch (SnappyError e) { + Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER); + Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer")); + } + } + + @Test + public void inputBufferSizeCheck() + throws Exception + { + ByteBuffer inputBuf = ByteBuffer.allocateDirect(9); + ByteBuffer outputBuf = ByteBuffer.allocateDirect(8); + + try { + BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf); + fail("no expected exception happened"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().equals("input length must be a multiple of a given type size")); + } + try { + BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf); + fail("no expected exception happened"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().equals("length of input shuffled data must be a multiple of a given type size")); + } + } + + @Test + public void outputBufferSizeCheck() + throws Exception { + ByteBuffer inputBuf = ByteBuffer.allocateDirect(12); + ByteBuffer outputBuf = ByteBuffer.allocateDirect(3); + + try { + BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf); + fail("no expected exception happened"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().equals("not enough space for output")); + } + try { + BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf); + fail("no expected exception happened"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().equals("not enough space for output")); + } + } + + @Test + public void bitShuffleInDirectLongArray() + throws Exception + { + ByteBuffer testData = ByteBuffer.allocateDirect(48); + ByteBuffer shuffled = ByteBuffer.allocateDirect(48); + testData.putLong(2); + testData.putLong(3); + testData.putLong(15); + testData.putLong(4234); + testData.putLong(43251531412342342L); + testData.putLong(23423422342L); + testData.flip(); + BitShuffle.bitShuffle(testData, BitShuffleType.LONG, shuffled); + ByteBuffer result = ByteBuffer.allocateDirect(48); + BitShuffle.bitUnShuffle(shuffled, BitShuffleType.LONG, result); + assertEquals(2L, result.getLong()); + assertEquals(3L, result.getLong()); + assertEquals(15L, result.getLong()); + assertEquals(4234L, result.getLong()); + assertEquals(43251531412342342L, result.getLong()); + assertEquals(23423422342L, result.getLong()); + } + + @Test + public void bitShuffleInDirectShortArray() + throws Exception + { + ByteBuffer testData = ByteBuffer.allocateDirect(18); + ByteBuffer shuffled = ByteBuffer.allocateDirect(18); + testData.putShort((short) 432); + testData.putShort((short) -32267); + testData.putShort((short) 1); + testData.putShort((short) 3); + testData.putShort((short) 34); + testData.putShort((short) 43); + testData.putShort((short) 34); + testData.putShort(Short.MAX_VALUE); + testData.putShort((short) -1); + testData.flip(); + BitShuffle.bitShuffle(testData, BitShuffleType.SHORT, shuffled); + ByteBuffer result = ByteBuffer.allocateDirect(18); + BitShuffle.bitUnShuffle(shuffled, BitShuffleType.SHORT, result); + assertEquals(432, result.getShort()); + assertEquals(-32267, result.getShort()); + assertEquals(1, result.getShort()); + assertEquals(3, result.getShort()); + assertEquals(34, result.getShort()); + assertEquals(43, result.getShort()); + assertEquals(34, result.getShort()); + assertEquals(Short.MAX_VALUE, result.getShort()); + assertEquals(-1, result.getShort()); + } + + @Test + public void bitShuffleInDirectIntArray() + throws Exception + { + ByteBuffer testData = ByteBuffer.allocateDirect(48); + ByteBuffer shuffled = ByteBuffer.allocateDirect(48); + testData.putInt(432); + testData.putInt(-32267); + testData.putInt(1); + testData.putInt(3); + testData.putInt(34); + testData.putInt(43); + testData.putInt(34); + testData.putInt(Short.MAX_VALUE); + testData.putInt(-1); + testData.putInt(Integer.MAX_VALUE); + testData.putInt(3424); + testData.putInt(43); + testData.flip(); + BitShuffle.bitShuffle(testData, BitShuffleType.INT, shuffled); + ByteBuffer result = ByteBuffer.allocateDirect(48); + BitShuffle.bitUnShuffle(shuffled, BitShuffleType.INT, result); + assertEquals(432, result.getInt()); + assertEquals(-32267, result.getInt()); + assertEquals(1, result.getInt()); + assertEquals(3, result.getInt()); + assertEquals(34, result.getInt()); + assertEquals(43, result.getInt()); + assertEquals(34, result.getInt()); + assertEquals(Short.MAX_VALUE, result.getInt()); + assertEquals(-1, result.getInt()); + assertEquals(Integer.MAX_VALUE, result.getInt()); + assertEquals(3424, result.getInt()); + assertEquals(43, result.getInt()); + } + + @Test + public void bitShuffleInDirectFloatArray() + throws Exception + { + ByteBuffer testData = ByteBuffer.allocateDirect(36); + ByteBuffer shuffled = ByteBuffer.allocateDirect(36); + testData.putFloat(100.0f); + testData.putFloat(0.5f); + testData.putFloat(-0.1f); + testData.putFloat(30.3f); + testData.putFloat(Float.MIN_NORMAL); + testData.putFloat(Float.MAX_EXPONENT); + testData.putFloat(Float.MAX_VALUE); + testData.putFloat(-0.1f); + testData.putFloat(Integer.MIN_VALUE); + testData.flip(); + BitShuffle.bitShuffle(testData, BitShuffleType.FLOAT, shuffled); + ByteBuffer result = ByteBuffer.allocateDirect(36); + BitShuffle.bitUnShuffle(shuffled, BitShuffleType.FLOAT, result); + assertEquals(100.0f, result.getFloat(), 0.0000001f); + assertEquals(0.5f, result.getFloat(), 0.0000001f); + assertEquals(-0.1f, result.getFloat(), 0.0000001f); + assertEquals(30.3f, result.getFloat(), 0.0000001f); + assertEquals(Float.MIN_NORMAL, result.getFloat(), 0.0000001f); + assertEquals(Float.MAX_EXPONENT, result.getFloat(), 0.0000001f); + assertEquals(Float.MAX_VALUE, result.getFloat(), 0.0000001f); + assertEquals(-0.1f, result.getFloat(), 0.0000001f); + assertEquals(Integer.MIN_VALUE, result.getFloat(), 0.0000001f); + } + + @Test + public void bitShuffleInDirectDoubleArray() + throws Exception + { + ByteBuffer testData = ByteBuffer.allocateDirect(72); + ByteBuffer shuffled = ByteBuffer.allocateDirect(72); + testData.putDouble(100.0); + testData.putDouble(0.5); + testData.putDouble(-0.1); + testData.putDouble(30.3); + testData.putDouble(Double.MIN_NORMAL); + testData.putDouble(Double.MAX_EXPONENT); + testData.putDouble(Double.MAX_VALUE); + testData.putDouble(-0.1); + testData.putDouble(Integer.MIN_VALUE); + testData.flip(); + BitShuffle.bitShuffle(testData, BitShuffleType.DOUBLE, shuffled); + ByteBuffer result = ByteBuffer.allocateDirect(72); + BitShuffle.bitUnShuffle(shuffled, BitShuffleType.DOUBLE, result); + assertEquals(100.0, result.getDouble(), 0.0000001); + assertEquals(0.5, result.getDouble(), 0.0000001); + assertEquals(-0.1, result.getDouble(), 0.0000001); + assertEquals(30.3, result.getDouble(), 0.0000001); + assertEquals(Double.MIN_NORMAL, result.getDouble(), 0.0000001); + assertEquals(Double.MAX_EXPONENT, result.getDouble(), 0.0000001); + assertEquals(Double.MAX_VALUE, result.getDouble(), 0.0000001); + assertEquals(-0.1, result.getDouble(), 0.0000001); + assertEquals(Integer.MIN_VALUE, result.getDouble(), 0.0000001); + } + @Test public void bitShuffleLongArray() throws Exception