Implement ByteBuffer-based APIs for BitShuffle (#155)

This commit is contained in:
Takeshi YAMAMURO 2017-01-25 23:26:23 +09:00
parent d127ab601a
commit 23f528095c
6 changed files with 439 additions and 2 deletions

View File

@ -25,6 +25,7 @@
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
public class BitShuffle
{
@ -42,6 +43,46 @@ public class BitShuffle
*/
private static BitShuffleNative impl;
/**
* Apply a bit-shuffling filter into the content in the given input buffer. After bit-shuffling,
* you can retrieve the shuffled data from the output buffer [pos() ...limit())
* (shuffled data size = limit() - pos() = remaining()).
*
* @param input buffer[pos() ... limit()) containing the input data
* @param type element type of the input data
* @param shuffled output of the shuffled data. Uses range [pos()..].
* @return byte size of the shuffled data.
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the input length is not a multiple of a given type size
*/
public static int bitShuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuffled) throws IOException {
if (!input.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = input.position();
int uLen = input.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("input length must be a multiple of a given type size");
}
if (shuffled.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.bitShuffleInDirectBuffer(input, uPos, typeSize, uLen, shuffled, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/**
* Apply a bit-shuffling filter into the input short array.
*
@ -112,6 +153,46 @@ public class BitShuffle
return output;
}
/**
* Convert the input bit-shuffled byte array into an original array. The result is dumped
* to the specified output buffer.
*
* @param shuffled buffer[pos() ... limit()) containing the input shuffled data
* @param type element type of the input data
* @param output output of the the original data. It uses buffer[pos()..]
* @return byte size of the unshuffled data.
* @throws IOException when failed to unshuffle the given input
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the length of input shuffled data is not a multiple of a given type size
*/
public static int bitUnShuffle(ByteBuffer shuffled, BitShuffleType type, ByteBuffer output) throws IOException {
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!output.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = shuffled.position();
int uLen = shuffled.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("length of input shuffled data must be a multiple of a given type size");
}
if (output.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.bitUnShuffleInDirectBuffer(shuffled, uPos, typeSize, uLen, output, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/**
* Convert the input bit-shuffled byte array into an original short array.
*

View File

@ -28,7 +28,7 @@ inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
}
/*
* Class: org_xerial_snappy_SnappyNative
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
@ -59,7 +59,28 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
}
/*
* Class: org_xerial_snappy_SnappyNative
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
@ -89,3 +110,24 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitunshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}

View File

@ -15,6 +15,14 @@ extern "C" {
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffle
@ -23,6 +31,14 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
#ifdef __cplusplus
}
#endif

View File

@ -25,6 +25,7 @@
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
@ -48,6 +49,12 @@ public class BitShuffleNative
public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
public native int bitShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
public native int bitUnShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
}

View File

@ -0,0 +1,53 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyErrorCode.java
// Since: 2011/03/30 14:56:50
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Type codes used in ByteBuffer based BitShuffle APIs
*
* @author leo
*/
public enum BitShuffleType
{
BYTE(1),
SHORT(2),
INT(4),
LONG(8),
FLOAT(4),
DOUBLE(8);
public final int id;
private BitShuffleType(int id)
{
this.id = id;
}
public int getTypeSize()
{
return id;
}
}

View File

@ -26,10 +26,248 @@ package org.xerial.snappy;
import static org.junit.Assert.*;
import java.nio.ByteBuffer;
import org.junit.Assert;
import org.junit.Test;
public class BitShuffleTest {
@Test
public void directBufferCheck()
throws Exception
{
ByteBuffer heapBuf = ByteBuffer.allocate(64);
ByteBuffer directBuf = ByteBuffer.allocateDirect(64);
// Tests for BitShuffle.bitShuffle()
try {
BitShuffle.bitShuffle(heapBuf, BitShuffleType.BYTE, directBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
}
try {
BitShuffle.bitShuffle(directBuf, BitShuffleType.BYTE, heapBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
}
// Then, tests for BitShuffle.bitUnShuffle()
try {
BitShuffle.bitUnShuffle(heapBuf, BitShuffleType.BYTE, directBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
}
try {
BitShuffle.bitUnShuffle(directBuf, BitShuffleType.BYTE, heapBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
}
}
@Test
public void inputBufferSizeCheck()
throws Exception
{
ByteBuffer inputBuf = ByteBuffer.allocateDirect(9);
ByteBuffer outputBuf = ByteBuffer.allocateDirect(8);
try {
BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("input length must be a multiple of a given type size"));
}
try {
BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("length of input shuffled data must be a multiple of a given type size"));
}
}
@Test
public void outputBufferSizeCheck()
throws Exception {
ByteBuffer inputBuf = ByteBuffer.allocateDirect(12);
ByteBuffer outputBuf = ByteBuffer.allocateDirect(3);
try {
BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
}
try {
BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
}
}
@Test
public void bitShuffleInDirectLongArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(48);
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
testData.putLong(2);
testData.putLong(3);
testData.putLong(15);
testData.putLong(4234);
testData.putLong(43251531412342342L);
testData.putLong(23423422342L);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.LONG, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(48);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.LONG, result);
assertEquals(2L, result.getLong());
assertEquals(3L, result.getLong());
assertEquals(15L, result.getLong());
assertEquals(4234L, result.getLong());
assertEquals(43251531412342342L, result.getLong());
assertEquals(23423422342L, result.getLong());
}
@Test
public void bitShuffleInDirectShortArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(18);
ByteBuffer shuffled = ByteBuffer.allocateDirect(18);
testData.putShort((short) 432);
testData.putShort((short) -32267);
testData.putShort((short) 1);
testData.putShort((short) 3);
testData.putShort((short) 34);
testData.putShort((short) 43);
testData.putShort((short) 34);
testData.putShort(Short.MAX_VALUE);
testData.putShort((short) -1);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.SHORT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(18);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.SHORT, result);
assertEquals(432, result.getShort());
assertEquals(-32267, result.getShort());
assertEquals(1, result.getShort());
assertEquals(3, result.getShort());
assertEquals(34, result.getShort());
assertEquals(43, result.getShort());
assertEquals(34, result.getShort());
assertEquals(Short.MAX_VALUE, result.getShort());
assertEquals(-1, result.getShort());
}
@Test
public void bitShuffleInDirectIntArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(48);
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
testData.putInt(432);
testData.putInt(-32267);
testData.putInt(1);
testData.putInt(3);
testData.putInt(34);
testData.putInt(43);
testData.putInt(34);
testData.putInt(Short.MAX_VALUE);
testData.putInt(-1);
testData.putInt(Integer.MAX_VALUE);
testData.putInt(3424);
testData.putInt(43);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.INT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(48);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.INT, result);
assertEquals(432, result.getInt());
assertEquals(-32267, result.getInt());
assertEquals(1, result.getInt());
assertEquals(3, result.getInt());
assertEquals(34, result.getInt());
assertEquals(43, result.getInt());
assertEquals(34, result.getInt());
assertEquals(Short.MAX_VALUE, result.getInt());
assertEquals(-1, result.getInt());
assertEquals(Integer.MAX_VALUE, result.getInt());
assertEquals(3424, result.getInt());
assertEquals(43, result.getInt());
}
@Test
public void bitShuffleInDirectFloatArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(36);
ByteBuffer shuffled = ByteBuffer.allocateDirect(36);
testData.putFloat(100.0f);
testData.putFloat(0.5f);
testData.putFloat(-0.1f);
testData.putFloat(30.3f);
testData.putFloat(Float.MIN_NORMAL);
testData.putFloat(Float.MAX_EXPONENT);
testData.putFloat(Float.MAX_VALUE);
testData.putFloat(-0.1f);
testData.putFloat(Integer.MIN_VALUE);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.FLOAT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(36);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.FLOAT, result);
assertEquals(100.0f, result.getFloat(), 0.0000001f);
assertEquals(0.5f, result.getFloat(), 0.0000001f);
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
assertEquals(30.3f, result.getFloat(), 0.0000001f);
assertEquals(Float.MIN_NORMAL, result.getFloat(), 0.0000001f);
assertEquals(Float.MAX_EXPONENT, result.getFloat(), 0.0000001f);
assertEquals(Float.MAX_VALUE, result.getFloat(), 0.0000001f);
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
assertEquals(Integer.MIN_VALUE, result.getFloat(), 0.0000001f);
}
@Test
public void bitShuffleInDirectDoubleArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(72);
ByteBuffer shuffled = ByteBuffer.allocateDirect(72);
testData.putDouble(100.0);
testData.putDouble(0.5);
testData.putDouble(-0.1);
testData.putDouble(30.3);
testData.putDouble(Double.MIN_NORMAL);
testData.putDouble(Double.MAX_EXPONENT);
testData.putDouble(Double.MAX_VALUE);
testData.putDouble(-0.1);
testData.putDouble(Integer.MIN_VALUE);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.DOUBLE, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(72);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.DOUBLE, result);
assertEquals(100.0, result.getDouble(), 0.0000001);
assertEquals(0.5, result.getDouble(), 0.0000001);
assertEquals(-0.1, result.getDouble(), 0.0000001);
assertEquals(30.3, result.getDouble(), 0.0000001);
assertEquals(Double.MIN_NORMAL, result.getDouble(), 0.0000001);
assertEquals(Double.MAX_EXPONENT, result.getDouble(), 0.0000001);
assertEquals(Double.MAX_VALUE, result.getDouble(), 0.0000001);
assertEquals(-0.1, result.getDouble(), 0.0000001);
assertEquals(Integer.MIN_VALUE, result.getDouble(), 0.0000001);
}
@Test
public void bitShuffleLongArray()
throws Exception