tyler.durden/snappy-java
1
0
Fork 0
mirror of https://github.com/xerial/snappy-java.git synced 2025-07-07 14:14:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement ByteBuffer-based APIs for BitShuffle (#155)

Browse Source
This commit is contained in:
Takeshi YAMAMURO 2017-01-25 23:26:23 +09:00
parent d127ab601a
commit 23f528095c
6 changed files with 439 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/main/java/org/xerial/snappy/BitShuffle.java
View File

@ -25,6 +25,7 @@
package org.xerial.snappy; package org.xerial.snappy;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer;
public class BitShuffle public class BitShuffle
{ {
@ -42,6 +43,46 @@ public class BitShuffle
*/ */
private static BitShuffleNative impl; private static BitShuffleNative impl;
/**
* Apply a bit-shuffling filter into the content in the given input buffer. After bit-shuffling,
* you can retrieve the shuffled data from the output buffer [pos() ...limit())
* (shuffled data size = limit() - pos() = remaining()).
*
* @param input buffer[pos() ... limit()) containing the input data
* @param type element type of the input data
* @param shuffled output of the shuffled data. Uses range [pos()..].
* @return byte size of the shuffled data.
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the input length is not a multiple of a given type size
*/
public static int bitShuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuffled) throws IOException {
if (!input.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = input.position();
int uLen = input.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("input length must be a multiple of a given type size");
}
if (shuffled.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.bitShuffleInDirectBuffer(input, uPos, typeSize, uLen, shuffled, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/** /**
* Apply a bit-shuffling filter into the input short array. * Apply a bit-shuffling filter into the input short array.
* *
@ -112,6 +153,46 @@ public class BitShuffle
return output; return output;
} }
/**
* Convert the input bit-shuffled byte array into an original array. The result is dumped
* to the specified output buffer.
*
* @param shuffled buffer[pos() ... limit()) containing the input shuffled data
* @param type element type of the input data
* @param output output of the the original data. It uses buffer[pos()..]
* @return byte size of the unshuffled data.
* @throws IOException when failed to unshuffle the given input
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the length of input shuffled data is not a multiple of a given type size
*/
public static int bitUnShuffle(ByteBuffer shuffled, BitShuffleType type, ByteBuffer output) throws IOException {
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!output.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = shuffled.position();
int uLen = shuffled.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("length of input shuffled data must be a multiple of a given type size");
}
if (output.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.bitUnShuffleInDirectBuffer(shuffled, uPos, typeSize, uLen, output, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/** /**
* Convert the input bit-shuffled byte array into an original short array. * Convert the input bit-shuffled byte array into an original short array.
* *

46
src/main/java/org/xerial/snappy/BitShuffleNative.cpp
View File

@ -28,7 +28,7 @@ inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
} }
/* /*
* Class: org_xerial_snappy_SnappyNative * Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffle * Method: bitShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/ */
@ -59,7 +59,28 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
} }
/* /*
* Class: org_xerial_snappy_SnappyNative * Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffle * Method: bitUnShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/ */
@ -89,3 +110,24 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
return (jint) processedBytes; return (jint) processedBytes;
} }
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitunshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}

16
src/main/java/org/xerial/snappy/BitShuffleNative.h
View File

@ -15,6 +15,14 @@ extern "C" {
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffleInDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/* /*
* Class: org_xerial_snappy_BitShuffleNative * Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffle * Method: bitUnShuffle
@ -23,6 +31,14 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffleInDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffleInDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
src/main/java/org/xerial/snappy/BitShuffleNative.java
View File

@ -25,6 +25,7 @@
package org.xerial.snappy; package org.xerial.snappy;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer;
/** /**
* JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is * JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
@ -48,6 +49,12 @@ public class BitShuffleNative
public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException; throws IOException;
public native int bitShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException; throws IOException;
public native int bitUnShuffleInDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
} }

53
src/main/java/org/xerial/snappy/BitShuffleType.java Normal file
View File

@ -0,0 +1,53 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyErrorCode.java
// Since: 2011/03/30 14:56:50
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Type codes used in ByteBuffer based BitShuffle APIs
*
* @author leo
*/
public enum BitShuffleType
{
BYTE(1),
SHORT(2),
INT(4),
LONG(8),
FLOAT(4),
DOUBLE(8);
public final int id;
private BitShuffleType(int id)
{
this.id = id;
}
public int getTypeSize()
{
return id;
}
}

238
src/test/java/org/xerial/snappy/BitShuffleTest.java
View File

@ -26,10 +26,248 @@ package org.xerial.snappy;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.nio.ByteBuffer;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
public class BitShuffleTest { public class BitShuffleTest {
@Test
public void directBufferCheck()
throws Exception
{
ByteBuffer heapBuf = ByteBuffer.allocate(64);
ByteBuffer directBuf = ByteBuffer.allocateDirect(64);
// Tests for BitShuffle.bitShuffle()
try {
BitShuffle.bitShuffle(heapBuf, BitShuffleType.BYTE, directBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
}
try {
BitShuffle.bitShuffle(directBuf, BitShuffleType.BYTE, heapBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
}
// Then, tests for BitShuffle.bitUnShuffle()
try {
BitShuffle.bitUnShuffle(heapBuf, BitShuffleType.BYTE, directBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
}
try {
BitShuffle.bitUnShuffle(directBuf, BitShuffleType.BYTE, heapBuf);
fail("no expected exception happened");
}
catch (SnappyError e) {
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
}
}
@Test
public void inputBufferSizeCheck()
throws Exception
{
ByteBuffer inputBuf = ByteBuffer.allocateDirect(9);
ByteBuffer outputBuf = ByteBuffer.allocateDirect(8);
try {
BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("input length must be a multiple of a given type size"));
}
try {
BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("length of input shuffled data must be a multiple of a given type size"));
}
}
@Test
public void outputBufferSizeCheck()
throws Exception {
ByteBuffer inputBuf = ByteBuffer.allocateDirect(12);
ByteBuffer outputBuf = ByteBuffer.allocateDirect(3);
try {
BitShuffle.bitShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
}
try {
BitShuffle.bitUnShuffle(inputBuf, BitShuffleType.INT, outputBuf);
fail("no expected exception happened");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
}
}
@Test
public void bitShuffleInDirectLongArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(48);
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
testData.putLong(2);
testData.putLong(3);
testData.putLong(15);
testData.putLong(4234);
testData.putLong(43251531412342342L);
testData.putLong(23423422342L);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.LONG, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(48);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.LONG, result);
assertEquals(2L, result.getLong());
assertEquals(3L, result.getLong());
assertEquals(15L, result.getLong());
assertEquals(4234L, result.getLong());
assertEquals(43251531412342342L, result.getLong());
assertEquals(23423422342L, result.getLong());
}
@Test
public void bitShuffleInDirectShortArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(18);
ByteBuffer shuffled = ByteBuffer.allocateDirect(18);
testData.putShort((short) 432);
testData.putShort((short) -32267);
testData.putShort((short) 1);
testData.putShort((short) 3);
testData.putShort((short) 34);
testData.putShort((short) 43);
testData.putShort((short) 34);
testData.putShort(Short.MAX_VALUE);
testData.putShort((short) -1);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.SHORT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(18);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.SHORT, result);
assertEquals(432, result.getShort());
assertEquals(-32267, result.getShort());
assertEquals(1, result.getShort());
assertEquals(3, result.getShort());
assertEquals(34, result.getShort());
assertEquals(43, result.getShort());
assertEquals(34, result.getShort());
assertEquals(Short.MAX_VALUE, result.getShort());
assertEquals(-1, result.getShort());
}
@Test
public void bitShuffleInDirectIntArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(48);
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
testData.putInt(432);
testData.putInt(-32267);
testData.putInt(1);
testData.putInt(3);
testData.putInt(34);
testData.putInt(43);
testData.putInt(34);
testData.putInt(Short.MAX_VALUE);
testData.putInt(-1);
testData.putInt(Integer.MAX_VALUE);
testData.putInt(3424);
testData.putInt(43);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.INT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(48);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.INT, result);
assertEquals(432, result.getInt());
assertEquals(-32267, result.getInt());
assertEquals(1, result.getInt());
assertEquals(3, result.getInt());
assertEquals(34, result.getInt());
assertEquals(43, result.getInt());
assertEquals(34, result.getInt());
assertEquals(Short.MAX_VALUE, result.getInt());
assertEquals(-1, result.getInt());
assertEquals(Integer.MAX_VALUE, result.getInt());
assertEquals(3424, result.getInt());
assertEquals(43, result.getInt());
}
@Test
public void bitShuffleInDirectFloatArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(36);
ByteBuffer shuffled = ByteBuffer.allocateDirect(36);
testData.putFloat(100.0f);
testData.putFloat(0.5f);
testData.putFloat(-0.1f);
testData.putFloat(30.3f);
testData.putFloat(Float.MIN_NORMAL);
testData.putFloat(Float.MAX_EXPONENT);
testData.putFloat(Float.MAX_VALUE);
testData.putFloat(-0.1f);
testData.putFloat(Integer.MIN_VALUE);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.FLOAT, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(36);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.FLOAT, result);
assertEquals(100.0f, result.getFloat(), 0.0000001f);
assertEquals(0.5f, result.getFloat(), 0.0000001f);
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
assertEquals(30.3f, result.getFloat(), 0.0000001f);
assertEquals(Float.MIN_NORMAL, result.getFloat(), 0.0000001f);
assertEquals(Float.MAX_EXPONENT, result.getFloat(), 0.0000001f);
assertEquals(Float.MAX_VALUE, result.getFloat(), 0.0000001f);
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
assertEquals(Integer.MIN_VALUE, result.getFloat(), 0.0000001f);
}
@Test
public void bitShuffleInDirectDoubleArray()
throws Exception
{
ByteBuffer testData = ByteBuffer.allocateDirect(72);
ByteBuffer shuffled = ByteBuffer.allocateDirect(72);
testData.putDouble(100.0);
testData.putDouble(0.5);
testData.putDouble(-0.1);
testData.putDouble(30.3);
testData.putDouble(Double.MIN_NORMAL);
testData.putDouble(Double.MAX_EXPONENT);
testData.putDouble(Double.MAX_VALUE);
testData.putDouble(-0.1);
testData.putDouble(Integer.MIN_VALUE);
testData.flip();
BitShuffle.bitShuffle(testData, BitShuffleType.DOUBLE, shuffled);
ByteBuffer result = ByteBuffer.allocateDirect(72);
BitShuffle.bitUnShuffle(shuffled, BitShuffleType.DOUBLE, result);
assertEquals(100.0, result.getDouble(), 0.0000001);
assertEquals(0.5, result.getDouble(), 0.0000001);
assertEquals(-0.1, result.getDouble(), 0.0000001);
assertEquals(30.3, result.getDouble(), 0.0000001);
assertEquals(Double.MIN_NORMAL, result.getDouble(), 0.0000001);
assertEquals(Double.MAX_EXPONENT, result.getDouble(), 0.0000001);
assertEquals(Double.MAX_VALUE, result.getDouble(), 0.0000001);
assertEquals(-0.1, result.getDouble(), 0.0000001);
assertEquals(Integer.MIN_VALUE, result.getDouble(), 0.0000001);
}
@Test @Test
public void bitShuffleLongArray() public void bitShuffleLongArray()
throws Exception throws Exception