Merge pull request #157 from maropu/ISSUE-155
Implement ByteBuffer-based APIs for BitShuffle
This commit is contained in:
Taro L. Saito
GitHub
commit
40b37ab343
|
|
@ -97,10 +97,10 @@ BitShuffle is an algorithm that reorders data bits (shuffle) for efficient compr
|
|||
import org.xerial.snappy.BitShuffle;
|
||||
|
||||
int[] data = new int[] {1, 3, 34, 43, 34};
|
||||
byte[] shuffledByteArray = BitShuffle.bitShuffle(data);
|
||||
byte[] shuffledByteArray = BitShuffle.shuffle(data);
|
||||
byte[] compressed = Snappy.compress(shuffledByteArray);
|
||||
byte[] uncompressed = Snappy.uncompress(compressed);
|
||||
int[] result = BitShuffle.bitUnShuffleIntArray(uncompress);
|
||||
int[] result = BitShuffle.unshuffleIntArray(uncompress);
|
||||
|
||||
System.out.println(result);
|
||||
```
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
package org.xerial.snappy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class BitShuffle
|
||||
{
|
||||
|
|
@ -42,6 +43,46 @@ public class BitShuffle
|
|||
*/
|
||||
private static BitShuffleNative impl;
|
||||
|
||||
/**
|
||||
* Apply a bit-shuffling filter into the content in the given input buffer. After bit-shuffling,
|
||||
* you can retrieve the shuffled data from the output buffer [pos() ...limit())
|
||||
* (shuffled data size = limit() - pos() = remaining()).
|
||||
*
|
||||
* @param input buffer[pos() ... limit()) containing the input data
|
||||
* @param type element type of the input data
|
||||
* @param shuffled output of the shuffled data. Uses range [pos()..].
|
||||
* @return byte size of the shuffled data.
|
||||
* @throws SnappyError when the input is not a direct buffer
|
||||
* @throws IllegalArgumentException when the input length is not a multiple of the given type size
|
||||
*/
|
||||
public static int shuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuffled) throws IOException {
|
||||
if (!input.isDirect()) {
|
||||
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
|
||||
}
|
||||
if (!shuffled.isDirect()) {
|
||||
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
|
||||
}
|
||||
|
||||
// input: input[pos(), limit())
|
||||
// output: shuffled
|
||||
int uPos = input.position();
|
||||
int uLen = input.remaining();
|
||||
int typeSize = type.getTypeSize();
|
||||
if (uLen % typeSize != 0) {
|
||||
throw new IllegalArgumentException("input length must be a multiple of the given type size: " + typeSize);
|
||||
}
|
||||
if (shuffled.remaining() < uLen) {
|
||||
throw new IllegalArgumentException("not enough space for output");
|
||||
}
|
||||
int numProcessed = impl.shuffleDirectBuffer(input, uPos, typeSize, uLen, shuffled, shuffled.position());
|
||||
assert(numProcessed == uLen);
|
||||
|
||||
// pos limit
|
||||
// [ ......BBBBBBB.........]
|
||||
shuffled.limit(shuffled.position() + numProcessed);
|
||||
return numProcessed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a bit-shuffling filter into the input short array.
|
||||
*
|
||||
|
|
@ -49,9 +90,9 @@ public class BitShuffle
|
|||
* @return bit-shuffled byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] bitShuffle(short[] input) throws IOException {
|
||||
public static byte[] shuffle(short[] input) throws IOException {
|
||||
byte[] output = new byte[input.length * 2];
|
||||
int numProcessed = impl.bitShuffle(input, 0, 2, input.length * 2, output, 0);
|
||||
int numProcessed = impl.shuffle(input, 0, 2, input.length * 2, output, 0);
|
||||
assert(numProcessed == input.length * 2);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -63,9 +104,9 @@ public class BitShuffle
|
|||
* @return bit-shuffled byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] bitShuffle(int[] input) throws IOException {
|
||||
public static byte[] shuffle(int[] input) throws IOException {
|
||||
byte[] output = new byte[input.length * 4];
|
||||
int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
|
||||
int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0);
|
||||
assert(numProcessed == input.length * 4);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -77,9 +118,9 @@ public class BitShuffle
|
|||
* @return bit-shuffled byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] bitShuffle(long[] input) throws IOException {
|
||||
public static byte[] shuffle(long[] input) throws IOException {
|
||||
byte[] output = new byte[input.length * 8];
|
||||
int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
|
||||
int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0);
|
||||
assert(numProcessed == input.length * 8);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -91,9 +132,9 @@ public class BitShuffle
|
|||
* @return bit-shuffled byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] bitShuffle(float[] input) throws IOException {
|
||||
public static byte[] shuffle(float[] input) throws IOException {
|
||||
byte[] output = new byte[input.length * 4];
|
||||
int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
|
||||
int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0);
|
||||
assert(numProcessed == input.length * 4);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -105,13 +146,53 @@ public class BitShuffle
|
|||
* @return bit-shuffled byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static byte[] bitShuffle(double[] input) throws IOException {
|
||||
public static byte[] shuffle(double[] input) throws IOException {
|
||||
byte[] output = new byte[input.length * 8];
|
||||
int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
|
||||
int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0);
|
||||
assert(numProcessed == input.length * 8);
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the input bit-shuffled byte array into an original array. The result is dumped
|
||||
* to the specified output buffer.
|
||||
*
|
||||
* @param shuffled buffer[pos() ... limit()) containing the input shuffled data
|
||||
* @param type element type of the input data
|
||||
* @param output output of the the original data. It uses buffer[pos()..]
|
||||
* @return byte size of the unshuffled data.
|
||||
* @throws IOException when failed to unshuffle the given input
|
||||
* @throws SnappyError when the input is not a direct buffer
|
||||
* @throws IllegalArgumentException when the length of input shuffled data is not a multiple of the given type size
|
||||
*/
|
||||
public static int unshuffle(ByteBuffer shuffled, BitShuffleType type, ByteBuffer output) throws IOException {
|
||||
if (!shuffled.isDirect()) {
|
||||
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
|
||||
}
|
||||
if (!output.isDirect()) {
|
||||
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
|
||||
}
|
||||
|
||||
// input: input[pos(), limit())
|
||||
// output: shuffled
|
||||
int uPos = shuffled.position();
|
||||
int uLen = shuffled.remaining();
|
||||
int typeSize = type.getTypeSize();
|
||||
if (uLen % typeSize != 0) {
|
||||
throw new IllegalArgumentException("length of input shuffled data must be a multiple of the given type size: " + typeSize);
|
||||
}
|
||||
if (output.remaining() < uLen) {
|
||||
throw new IllegalArgumentException("not enough space for output");
|
||||
}
|
||||
int numProcessed = impl.unshuffleDirectBuffer(shuffled, uPos, typeSize, uLen, output, shuffled.position());
|
||||
assert(numProcessed == uLen);
|
||||
|
||||
// pos limit
|
||||
// [ ......BBBBBBB.........]
|
||||
shuffled.limit(shuffled.position() + numProcessed);
|
||||
return numProcessed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the input bit-shuffled byte array into an original short array.
|
||||
*
|
||||
|
|
@ -119,9 +200,9 @@ public class BitShuffle
|
|||
* @return a short array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static short[] bitUnShuffleShortArray(byte[] input) throws IOException {
|
||||
public static short[] unshuffleShortArray(byte[] input) throws IOException {
|
||||
short[] output = new short[input.length / 2];
|
||||
int numProcessed = impl.bitUnShuffle(input, 0, 2, input.length, output, 0);
|
||||
int numProcessed = impl.unshuffle(input, 0, 2, input.length, output, 0);
|
||||
assert(numProcessed == input.length);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -133,9 +214,9 @@ public class BitShuffle
|
|||
* @return an int array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static int[] bitUnShuffleIntArray(byte[] input) throws IOException {
|
||||
public static int[] unshuffleIntArray(byte[] input) throws IOException {
|
||||
int[] output = new int[input.length / 4];
|
||||
int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
|
||||
int numProcessed = impl.unshuffle(input, 0, 4, input.length, output, 0);
|
||||
assert(numProcessed == input.length);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -147,9 +228,9 @@ public class BitShuffle
|
|||
* @return a long array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static long[] bitUnShuffleLongArray(byte[] input) throws IOException {
|
||||
public static long[] unshuffleLongArray(byte[] input) throws IOException {
|
||||
long[] output = new long[input.length / 8];
|
||||
int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
|
||||
int numProcessed = impl.unshuffle(input, 0, 8, input.length, output, 0);
|
||||
assert(numProcessed == input.length);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -161,9 +242,9 @@ public class BitShuffle
|
|||
* @return an float array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static float[] bitUnShuffleFloatArray(byte[] input) throws IOException {
|
||||
public static float[] unshuffleFloatArray(byte[] input) throws IOException {
|
||||
float[] output = new float[input.length / 4];
|
||||
int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
|
||||
int numProcessed = impl.unshuffle(input, 0, 4, input.length, output, 0);
|
||||
assert(numProcessed == input.length);
|
||||
return output;
|
||||
}
|
||||
|
|
@ -175,9 +256,9 @@ public class BitShuffle
|
|||
* @return a double array
|
||||
* @throws IOException
|
||||
*/
|
||||
public static double[] bitUnShuffleDoubleArray(byte[] input) throws IOException {
|
||||
public static double[] unshuffleDoubleArray(byte[] input) throws IOException {
|
||||
double[] output = new double[input.length / 8];
|
||||
int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
|
||||
int numProcessed = impl.unshuffle(input, 0, 8, input.length, output, 0);
|
||||
assert(numProcessed == input.length);
|
||||
return output;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,11 +28,11 @@ inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
|
|||
}
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_SnappyNative
|
||||
* Method: bitShuffle
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: shuffle
|
||||
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffle
|
||||
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
|
||||
{
|
||||
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
|
||||
|
|
@ -59,11 +59,32 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
|
|||
}
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_SnappyNative
|
||||
* Method: bitUnShuffle
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: shuffleDirectBuffer
|
||||
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffleDirectBuffer
|
||||
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
|
||||
{
|
||||
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
|
||||
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
|
||||
if(inputBuffer == 0 || outputBuffer == 0) {
|
||||
throw_exception(env, self, 3);
|
||||
return (jint) 0;
|
||||
}
|
||||
|
||||
int64_t processedBytes = bshuf_bitshuffle(
|
||||
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
|
||||
|
||||
return (jint) processedBytes;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: unshuffle
|
||||
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffle
|
||||
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
|
||||
{
|
||||
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
|
||||
|
|
@ -89,3 +110,24 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
|
|||
return (jint) processedBytes;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: unshuffleDirectBuffer
|
||||
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffleDirectBuffer
|
||||
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
|
||||
{
|
||||
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
|
||||
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
|
||||
if(inputBuffer == 0 || outputBuffer == 0) {
|
||||
throw_exception(env, self, 3);
|
||||
return (jint) 0;
|
||||
}
|
||||
|
||||
int64_t processedBytes = bshuf_bitunshuffle(
|
||||
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
|
||||
|
||||
return (jint) processedBytes;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,18 +9,34 @@ extern "C" {
|
|||
#endif
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: bitShuffle
|
||||
* Method: shuffle
|
||||
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffle
|
||||
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: bitUnShuffle
|
||||
* Method: shuffleDirectBuffer
|
||||
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffleDirectBuffer
|
||||
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: unshuffle
|
||||
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffle
|
||||
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
|
||||
|
||||
/*
|
||||
* Class: org_xerial_snappy_BitShuffleNative
|
||||
* Method: unshuffleDirectBuffer
|
||||
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
|
||||
*/
|
||||
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffleDirectBuffer
|
||||
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
package org.xerial.snappy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
|
||||
|
|
@ -45,9 +46,15 @@ public class BitShuffleNative
|
|||
// A quick benchmark result can be found in a gist below;
|
||||
// https://gist.github.com/maropu/01103215df34b317a7a7
|
||||
// ------------------------------------------------------------------------
|
||||
public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
|
||||
public native int shuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
|
||||
throws IOException;
|
||||
|
||||
public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
|
||||
public native int shuffleDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
|
||||
throws IOException;
|
||||
|
||||
public native int unshuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
|
||||
throws IOException;
|
||||
|
||||
public native int unshuffleDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
|
||||
throws IOException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
/*--------------------------------------------------------------------------
|
||||
* Copyright 2011 Taro L. Saito
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*--------------------------------------------------------------------------*/
|
||||
//--------------------------------------
|
||||
// XerialJ
|
||||
//
|
||||
// SnappyErrorCode.java
|
||||
// Since: 2011/03/30 14:56:50
|
||||
//
|
||||
// $URL$
|
||||
// $Author$
|
||||
//--------------------------------------
|
||||
package org.xerial.snappy;
|
||||
|
||||
/**
|
||||
* Type codes used in ByteBuffer based BitShuffle APIs
|
||||
*
|
||||
* @author leo
|
||||
*/
|
||||
public enum BitShuffleType
|
||||
{
|
||||
|
||||
BYTE(1),
|
||||
SHORT(2),
|
||||
INT(4),
|
||||
LONG(8),
|
||||
FLOAT(4),
|
||||
DOUBLE(8);
|
||||
|
||||
public final int id;
|
||||
|
||||
private BitShuffleType(int id)
|
||||
{
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public int getTypeSize()
|
||||
{
|
||||
return id;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -26,57 +26,295 @@ package org.xerial.snappy;
|
|||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class BitShuffleTest {
|
||||
|
||||
@Test
|
||||
public void bitShuffleLongArray()
|
||||
public void directBufferCheck()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer heapBuf = ByteBuffer.allocate(64);
|
||||
ByteBuffer directBuf = ByteBuffer.allocateDirect(64);
|
||||
|
||||
// Tests for BitShuffle.shuffle()
|
||||
try {
|
||||
BitShuffle.shuffle(heapBuf, BitShuffleType.BYTE, directBuf);
|
||||
fail("no expected exception happened");
|
||||
}
|
||||
catch (SnappyError e) {
|
||||
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
|
||||
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
|
||||
}
|
||||
try {
|
||||
BitShuffle.shuffle(directBuf, BitShuffleType.BYTE, heapBuf);
|
||||
fail("no expected exception happened");
|
||||
}
|
||||
catch (SnappyError e) {
|
||||
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
|
||||
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
|
||||
}
|
||||
|
||||
// Then, tests for BitShuffle.unshuffle()
|
||||
try {
|
||||
BitShuffle.unshuffle(heapBuf, BitShuffleType.BYTE, directBuf);
|
||||
fail("no expected exception happened");
|
||||
}
|
||||
catch (SnappyError e) {
|
||||
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
|
||||
Assert.assertTrue(e.getMessage().contains("input is not a direct buffer"));
|
||||
}
|
||||
try {
|
||||
BitShuffle.unshuffle(directBuf, BitShuffleType.BYTE, heapBuf);
|
||||
fail("no expected exception happened");
|
||||
}
|
||||
catch (SnappyError e) {
|
||||
Assert.assertTrue(e.errorCode == SnappyErrorCode.NOT_A_DIRECT_BUFFER);
|
||||
Assert.assertTrue(e.getMessage().contains("destination is not a direct buffer"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void inputBufferSizeCheck()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer inputBuf = ByteBuffer.allocateDirect(9);
|
||||
ByteBuffer outputBuf = ByteBuffer.allocateDirect(8);
|
||||
|
||||
try {
|
||||
BitShuffle.shuffle(inputBuf, BitShuffleType.INT, outputBuf);
|
||||
fail("no expected exception happened");
|
||||
} catch (IllegalArgumentException e) {
|
||||
Assert.assertTrue(e.getMessage().startsWith("input length must be a multiple of the given type size"));
|
||||
}
|
||||
try {
|
||||
BitShuffle.unshuffle(inputBuf, BitShuffleType.INT, outputBuf);
|
||||
fail("no expected exception happened");
|
||||
} catch (IllegalArgumentException e) {
|
||||
Assert.assertTrue(e.getMessage().startsWith("length of input shuffled data must be a multiple of the given type size"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void outputBufferSizeCheck()
|
||||
throws Exception {
|
||||
ByteBuffer inputBuf = ByteBuffer.allocateDirect(12);
|
||||
ByteBuffer outputBuf = ByteBuffer.allocateDirect(3);
|
||||
|
||||
try {
|
||||
BitShuffle.shuffle(inputBuf, BitShuffleType.INT, outputBuf);
|
||||
fail("no expected exception happened");
|
||||
} catch (IllegalArgumentException e) {
|
||||
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
|
||||
}
|
||||
try {
|
||||
BitShuffle.unshuffle(inputBuf, BitShuffleType.INT, outputBuf);
|
||||
fail("no expected exception happened");
|
||||
} catch (IllegalArgumentException e) {
|
||||
Assert.assertTrue(e.getMessage().equals("not enough space for output"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleDirectLongArray()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer testData = ByteBuffer.allocateDirect(48);
|
||||
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
|
||||
testData.putLong(2);
|
||||
testData.putLong(3);
|
||||
testData.putLong(15);
|
||||
testData.putLong(4234);
|
||||
testData.putLong(43251531412342342L);
|
||||
testData.putLong(23423422342L);
|
||||
testData.flip();
|
||||
BitShuffle.shuffle(testData, BitShuffleType.LONG, shuffled);
|
||||
ByteBuffer result = ByteBuffer.allocateDirect(48);
|
||||
BitShuffle.unshuffle(shuffled, BitShuffleType.LONG, result);
|
||||
assertEquals(2L, result.getLong());
|
||||
assertEquals(3L, result.getLong());
|
||||
assertEquals(15L, result.getLong());
|
||||
assertEquals(4234L, result.getLong());
|
||||
assertEquals(43251531412342342L, result.getLong());
|
||||
assertEquals(23423422342L, result.getLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleDirectShortArray()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer testData = ByteBuffer.allocateDirect(18);
|
||||
ByteBuffer shuffled = ByteBuffer.allocateDirect(18);
|
||||
testData.putShort((short) 432);
|
||||
testData.putShort((short) -32267);
|
||||
testData.putShort((short) 1);
|
||||
testData.putShort((short) 3);
|
||||
testData.putShort((short) 34);
|
||||
testData.putShort((short) 43);
|
||||
testData.putShort((short) 34);
|
||||
testData.putShort(Short.MAX_VALUE);
|
||||
testData.putShort((short) -1);
|
||||
testData.flip();
|
||||
BitShuffle.shuffle(testData, BitShuffleType.SHORT, shuffled);
|
||||
ByteBuffer result = ByteBuffer.allocateDirect(18);
|
||||
BitShuffle.unshuffle(shuffled, BitShuffleType.SHORT, result);
|
||||
assertEquals(432, result.getShort());
|
||||
assertEquals(-32267, result.getShort());
|
||||
assertEquals(1, result.getShort());
|
||||
assertEquals(3, result.getShort());
|
||||
assertEquals(34, result.getShort());
|
||||
assertEquals(43, result.getShort());
|
||||
assertEquals(34, result.getShort());
|
||||
assertEquals(Short.MAX_VALUE, result.getShort());
|
||||
assertEquals(-1, result.getShort());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleDirectIntArray()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer testData = ByteBuffer.allocateDirect(48);
|
||||
ByteBuffer shuffled = ByteBuffer.allocateDirect(48);
|
||||
testData.putInt(432);
|
||||
testData.putInt(-32267);
|
||||
testData.putInt(1);
|
||||
testData.putInt(3);
|
||||
testData.putInt(34);
|
||||
testData.putInt(43);
|
||||
testData.putInt(34);
|
||||
testData.putInt(Short.MAX_VALUE);
|
||||
testData.putInt(-1);
|
||||
testData.putInt(Integer.MAX_VALUE);
|
||||
testData.putInt(3424);
|
||||
testData.putInt(43);
|
||||
testData.flip();
|
||||
BitShuffle.shuffle(testData, BitShuffleType.INT, shuffled);
|
||||
ByteBuffer result = ByteBuffer.allocateDirect(48);
|
||||
BitShuffle.unshuffle(shuffled, BitShuffleType.INT, result);
|
||||
assertEquals(432, result.getInt());
|
||||
assertEquals(-32267, result.getInt());
|
||||
assertEquals(1, result.getInt());
|
||||
assertEquals(3, result.getInt());
|
||||
assertEquals(34, result.getInt());
|
||||
assertEquals(43, result.getInt());
|
||||
assertEquals(34, result.getInt());
|
||||
assertEquals(Short.MAX_VALUE, result.getInt());
|
||||
assertEquals(-1, result.getInt());
|
||||
assertEquals(Integer.MAX_VALUE, result.getInt());
|
||||
assertEquals(3424, result.getInt());
|
||||
assertEquals(43, result.getInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleDirectFloatArray()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer testData = ByteBuffer.allocateDirect(36);
|
||||
ByteBuffer shuffled = ByteBuffer.allocateDirect(36);
|
||||
testData.putFloat(100.0f);
|
||||
testData.putFloat(0.5f);
|
||||
testData.putFloat(-0.1f);
|
||||
testData.putFloat(30.3f);
|
||||
testData.putFloat(Float.MIN_NORMAL);
|
||||
testData.putFloat(Float.MAX_EXPONENT);
|
||||
testData.putFloat(Float.MAX_VALUE);
|
||||
testData.putFloat(-0.1f);
|
||||
testData.putFloat(Integer.MIN_VALUE);
|
||||
testData.flip();
|
||||
BitShuffle.shuffle(testData, BitShuffleType.FLOAT, shuffled);
|
||||
ByteBuffer result = ByteBuffer.allocateDirect(36);
|
||||
BitShuffle.unshuffle(shuffled, BitShuffleType.FLOAT, result);
|
||||
assertEquals(100.0f, result.getFloat(), 0.0000001f);
|
||||
assertEquals(0.5f, result.getFloat(), 0.0000001f);
|
||||
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
|
||||
assertEquals(30.3f, result.getFloat(), 0.0000001f);
|
||||
assertEquals(Float.MIN_NORMAL, result.getFloat(), 0.0000001f);
|
||||
assertEquals(Float.MAX_EXPONENT, result.getFloat(), 0.0000001f);
|
||||
assertEquals(Float.MAX_VALUE, result.getFloat(), 0.0000001f);
|
||||
assertEquals(-0.1f, result.getFloat(), 0.0000001f);
|
||||
assertEquals(Integer.MIN_VALUE, result.getFloat(), 0.0000001f);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleDirectDoubleArray()
|
||||
throws Exception
|
||||
{
|
||||
ByteBuffer testData = ByteBuffer.allocateDirect(72);
|
||||
ByteBuffer shuffled = ByteBuffer.allocateDirect(72);
|
||||
testData.putDouble(100.0);
|
||||
testData.putDouble(0.5);
|
||||
testData.putDouble(-0.1);
|
||||
testData.putDouble(30.3);
|
||||
testData.putDouble(Double.MIN_NORMAL);
|
||||
testData.putDouble(Double.MAX_EXPONENT);
|
||||
testData.putDouble(Double.MAX_VALUE);
|
||||
testData.putDouble(-0.1);
|
||||
testData.putDouble(Integer.MIN_VALUE);
|
||||
testData.flip();
|
||||
BitShuffle.shuffle(testData, BitShuffleType.DOUBLE, shuffled);
|
||||
ByteBuffer result = ByteBuffer.allocateDirect(72);
|
||||
BitShuffle.unshuffle(shuffled, BitShuffleType.DOUBLE, result);
|
||||
assertEquals(100.0, result.getDouble(), 0.0000001);
|
||||
assertEquals(0.5, result.getDouble(), 0.0000001);
|
||||
assertEquals(-0.1, result.getDouble(), 0.0000001);
|
||||
assertEquals(30.3, result.getDouble(), 0.0000001);
|
||||
assertEquals(Double.MIN_NORMAL, result.getDouble(), 0.0000001);
|
||||
assertEquals(Double.MAX_EXPONENT, result.getDouble(), 0.0000001);
|
||||
assertEquals(Double.MAX_VALUE, result.getDouble(), 0.0000001);
|
||||
assertEquals(-0.1, result.getDouble(), 0.0000001);
|
||||
assertEquals(Integer.MIN_VALUE, result.getDouble(), 0.0000001);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shuffleLongArray()
|
||||
throws Exception
|
||||
{
|
||||
long[] data = new long[] {2, 3, 15, 4234, 43251531412342342L, 23423422342L};
|
||||
byte[] shuffledData = BitShuffle.bitShuffle(data);
|
||||
long[] result = BitShuffle.bitUnShuffleLongArray(shuffledData);
|
||||
byte[] shuffledData = BitShuffle.shuffle(data);
|
||||
long[] result = BitShuffle.unshuffleLongArray(shuffledData);
|
||||
assertArrayEquals(data, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bitShuffleShortArray()
|
||||
public void shuffleShortArray()
|
||||
throws Exception
|
||||
{
|
||||
short[] data = new short[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1};
|
||||
byte[] shuffledData = BitShuffle.bitShuffle(data);
|
||||
short[] result = BitShuffle.bitUnShuffleShortArray(shuffledData);
|
||||
byte[] shuffledData = BitShuffle.shuffle(data);
|
||||
short[] result = BitShuffle.unshuffleShortArray(shuffledData);
|
||||
assertArrayEquals(data, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bitShuffleIntArray()
|
||||
public void shuffleIntArray()
|
||||
throws Exception
|
||||
{
|
||||
int[] data = new int[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43};
|
||||
byte[] shuffledData = BitShuffle.bitShuffle(data);
|
||||
int[] result = BitShuffle.bitUnShuffleIntArray(shuffledData);
|
||||
byte[] shuffledData = BitShuffle.shuffle(data);
|
||||
int[] result = BitShuffle.unshuffleIntArray(shuffledData);
|
||||
assertArrayEquals(data, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bitShuffleFloatArray()
|
||||
public void shuffleFloatArray()
|
||||
throws Exception
|
||||
{
|
||||
float[] data = new float[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
|
||||
byte[] shuffledData = BitShuffle.bitShuffle(data);
|
||||
float[] result = BitShuffle.bitUnShuffleFloatArray(shuffledData);
|
||||
byte[] shuffledData = BitShuffle.shuffle(data);
|
||||
float[] result = BitShuffle.unshuffleFloatArray(shuffledData);
|
||||
assertArrayEquals(data, result, 0.0000001f);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bitShuffleDoubleArray()
|
||||
public void shuffleDoubleArray()
|
||||
throws Exception
|
||||
{
|
||||
double[] data = new double[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
|
||||
byte[] shuffledData = BitShuffle.bitShuffle(data);
|
||||
double[] result = BitShuffle.bitUnShuffleDoubleArray(shuffledData);
|
||||
byte[] shuffledData = BitShuffle.shuffle(data);
|
||||
double[] result = BitShuffle.unshuffleDoubleArray(shuffledData);
|
||||
assertArrayEquals(data, result, 0.0000001f);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue