Merge pull request #153 from xerial/revert-152-revert-137-SupportBitShuffle

Revert "Revert "[WIP] Add bit-shuffling interfaces to improve LZ performance""

Makefile

@@ -7,17 +7,50 @@ SBT:=./sbt
 all: snappy
 
 SNAPPY_OUT:=$(TARGET)/$(snappy)-$(os_arch)
-SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(VERSION).tar.gz 
+SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz
 SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
-SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(VERSION)
+SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
 SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
-SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) SnappyNative.o)
-
-SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
 SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
 SNAPPY_GIT_REV:=2b9152d9c5bed71dffb7f7f6c7a3ec48b058ff2d # 1.1.3 with autogen.sh fix
+SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
+SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
 
-CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR)
+BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz
+BITSHUFFLE_C:=bitshuffle_core.c iochain.c
+BITSHUFFLE_SRC_DIR:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION)/src
+BITSHUFFLE_SRC:=$(addprefix $(BITSHUFFLE_SRC_DIR)/,$(BITSHUFFLE_C))
+BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log
+
+$(BITSHUFFLE_ARCHIVE):
+	@mkdir -p $(@D)
+	curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz
+
+$(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE)
+	$(TAR) xvfz $< -C $(TARGET)
+	touch $@
+
+$(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED)
+
+$(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c
+	@mkdir -p $(@D)
+	$(CXX) $(CXXFLAGS) $(CXXFLAGS_BITSHUFFLE) -c $< -o $@
+
+SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o)
+
+CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(BITSHUFFLE_SRC_DIR)
+
+ifndef CXXFLAGS_BITSHUFFLE
+  ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64)
+	# SSE2 is supported in all the x86_64 platforms and AVX2 is only supported
+        # in the small part of them. gcc in linux/x86_64 typically enables SSE2 by default though,
+	# we explicitly set flags below to make this precondition clearer.
+	CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -msse2
+  else
+	# Undefined macros to generate a platform-independent binary
+	CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -U__SSE2__
+  endif
+endif
 
 ifeq ($(OS_NAME),SunOS)
 	TAR:= gtar
@@ -27,7 +60,12 @@ endif
 
 $(SNAPPY_ARCHIVE):
 	@mkdir -p $(@D)
-	curl -L -o$@ https://github.com/google/snappy/releases/download/$(VERSION)/snappy-$(VERSION).tar.gz
+	curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz
+
+$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
+	$(TAR) xvfz $< -C $(TARGET)
+	touch $@
+	cd  $(SNAPPY_SRC_DIR) && ./configure
 
 $(SNAPPY_GIT_UNPACKED):
 	rm -rf $(SNAPPY_SRC_DIR)
@@ -37,25 +75,35 @@ $(SNAPPY_GIT_UNPACKED):
 	cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure
 	touch $@
 
-jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h
+jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
 
-$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/snappy/SnappyNative.java
+$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java
 	@mkdir -p $(TARGET)/jni-classes
 	$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
 
 $(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class
 	$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative
 
+$(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java
+	@mkdir -p $(TARGET)/jni-classes
+	$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
+
+$(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class
+	$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.BitShuffleNative
+
 $(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
 
-$(SNAPPY_OUT)/%.o : $(SNAPPY_SRC_DIR)/%.cc
+$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc
 	@mkdir -p $(@D)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
-$(SNAPPY_OUT)/SnappyNative.o : $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h  
+$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
 	@mkdir -p $(@D)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h
+	@mkdir -p $(@D)
+	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 $(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ)
 	$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
@@ -133,17 +181,16 @@ install-m2:
 
 googlecode-upload: googlecode-lib-upload googlecode-src-upload
 
-googlecode-lib-upload: $(TARGET)/snappy-java-$(VERSION)-lib.upload
+googlecode-lib-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload
-googlecode-src-upload: $(TARGET)/snappy-java-$(VERSION)-src.upload
+googlecode-src-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload
 
 GOOGLECODE_USER:=leo@xerial.org
 
-$(TARGET)/snappy-java-$(VERSION)-lib.upload:
+$(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload:
-	./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).jar 
+	./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).jar
 	touch $@
 
-$(TARGET)/snappy-java-$(VERSION)-src.upload:
+$(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload:
-	./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).tar.gz 
+	./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).tar.gz
 	touch $@
 
-

Makefile.common

@@ -31,7 +31,7 @@ else
 	sep := :
 endif
 
-snappy := snappy-$(VERSION)
+snappy := snappy-$(SNAPPY_VERSION)
 
 
 jni_md := $(shell find -L "$(JAVA_HOME)" -name jni_md.h | head -1)
@@ -257,4 +257,3 @@ ifneq ($(jni_include),)
 CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
 endif
 
-

README.md

@@ -143,6 +143,12 @@ Older snapshots of snappy contain a buggy config.h.in that does not work properl
 	$ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x
 	$ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1
 
+## Activating SSE2/AVX2 instructions in BitShuffle
+
+The most of the native libraries that snappy-java contains disable SSE2/AVX2 instructions in terms of portability (SSE2 is enabled only in Linux/x86_64 platforms). To enable AVX2 instructions, you need to compile as follows:
+
+	$ make CXXFLAGS_BITSHUFFLE=-mavx2  # -msse2 for SSE2 instructions
+
 ## Cross-compiling for other platforms
 The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM:
 

src/main/java/org/xerial/snappy/BitShuffle.java

@@ -0,0 +1,184 @@
+/*--------------------------------------------------------------------------
+ *  Copyright 2011 Taro L. Saito
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *--------------------------------------------------------------------------*/
+//--------------------------------------
+// snappy-java Project
+//
+// BitShuffle.java
+// Since: 2016/03/31
+//
+// $URL$
+// $Author$
+//--------------------------------------
+package org.xerial.snappy;
+
+import java.io.IOException;
+
+public class BitShuffle
+{
+    static {
+        try {
+            impl = SnappyLoader.loadBitShuffleApi();
+        }
+        catch (Exception e) {
+            throw new ExceptionInInitializerError(e);
+        }
+    }
+
+    /**
+     * An instance of BitShuffleNative
+     */
+    private static BitShuffleNative impl;
+
+    /**
+     * Apply a bit-shuffling filter into the input short array.
+     *
+     * @param input
+     * @return bit-shuffled byte array
+     * @throws IOException
+     */
+    public static byte[] bitShuffle(short[] input) throws IOException {
+        byte[] output = new byte[input.length * 2];
+        int numProcessed = impl.bitShuffle(input, 0, 2, input.length * 2, output, 0);
+        assert(numProcessed == input.length * 2);
+        return output;
+    }
+
+    /**
+     * Apply a bit-shuffling filter into the input int array.
+     *
+     * @param input
+     * @return bit-shuffled byte array
+     * @throws IOException
+     */
+    public static byte[] bitShuffle(int[] input) throws IOException {
+        byte[] output = new byte[input.length * 4];
+        int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
+        assert(numProcessed == input.length * 4);
+        return output;
+    }
+
+    /**
+     * Apply a bit-shuffling filter into the input long array.
+     *
+     * @param input
+     * @return bit-shuffled byte array
+     * @throws IOException
+     */
+    public static byte[] bitShuffle(long[] input) throws IOException {
+        byte[] output = new byte[input.length * 8];
+        int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
+        assert(numProcessed == input.length * 8);
+        return output;
+    }
+
+    /**
+     * Apply a bit-shuffling filter into the input float array.
+     *
+     * @param input
+     * @return bit-shuffled byte array
+     * @throws IOException
+     */
+    public static byte[] bitShuffle(float[] input) throws IOException {
+        byte[] output = new byte[input.length * 4];
+        int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
+        assert(numProcessed == input.length * 4);
+        return output;
+    }
+
+    /**
+     * Apply a bit-shuffling filter into the input double array.
+     *
+     * @param input
+     * @return bit-shuffled byte array
+     * @throws IOException
+     */
+    public static byte[] bitShuffle(double[] input) throws IOException {
+        byte[] output = new byte[input.length * 8];
+        int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
+        assert(numProcessed == input.length * 8);
+        return output;
+    }
+
+    /**
+     * Convert the input bit-shuffled byte array into an original short array.
+     *
+     * @param input
+     * @return a short array
+     * @throws IOException
+     */
+    public static short[] bitUnShuffleShortArray(byte[] input) throws IOException {
+        short[] output = new short[input.length / 2];
+        int numProcessed = impl.bitUnShuffle(input, 0, 2, input.length, output, 0);
+        assert(numProcessed == input.length);
+        return output;
+    }
+
+    /**
+     * Convert the input bit-shuffled byte array into an original int array.
+     *
+     * @param input
+     * @return an int array
+     * @throws IOException
+     */
+    public static int[] bitUnShuffleIntArray(byte[] input) throws IOException {
+        int[] output = new int[input.length / 4];
+        int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
+        assert(numProcessed == input.length);
+        return output;
+    }
+
+    /**
+     * Convert the input bit-shuffled byte array into an original long array.
+     *
+     * @param input
+     * @return a long array
+     * @throws IOException
+     */
+    public static long[] bitUnShuffleLongArray(byte[] input) throws IOException {
+        long[] output = new long[input.length / 8];
+        int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
+        assert(numProcessed == input.length);
+        return output;
+    }
+
+    /**
+     * Convert the input bit-shuffled byte array into an original float array.
+     *
+     * @param input
+     * @return an float array
+     * @throws IOException
+     */
+    public static float[] bitUnShuffleFloatArray(byte[] input) throws IOException {
+        float[] output = new float[input.length / 4];
+        int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
+        assert(numProcessed == input.length);
+        return output;
+    }
+
+    /**
+     * Convert the input bit-shuffled byte array into an original double array.
+     *
+     * @param input
+     * @return a double array
+     * @throws IOException
+     */
+    public static double[] bitUnShuffleDoubleArray(byte[] input) throws IOException {
+        double[] output = new double[input.length / 8];
+        int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
+        assert(numProcessed == input.length);
+        return output;
+    }
+}

src/main/java/org/xerial/snappy/BitShuffleNative.cpp

@@ -0,0 +1,91 @@
+/*--------------------------------------------------------------------------
+ *  Copyright 2011 Taro L. Saito
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *--------------------------------------------------------------------------*/
+#include <bitshuffle.h>
+#include "BitShuffleNative.h"
+
+inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
+{
+	jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
+	if(c==0)
+		return;
+	jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
+	if(mth_throwex == 0)
+		return;
+	env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
+}
+
+/*
+ * Class:     org_xerial_snappy_SnappyNative
+ * Method:    bitShuffle
+ * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
+ */
+JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
+  (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
+{
+	char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
+	char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
+	if(in == 0 || out == 0) {
+		// out of memory
+		if(in != 0) {
+			env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
+		}
+		if(out != 0) {
+			env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
+		}
+		throw_exception(env, self, 4);
+		return 0;
+	}
+
+        int64_t processedBytes = bshuf_bitshuffle(
+                        in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
+
+	env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
+	env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
+
+	return (jint) processedBytes;
+}
+
+/*
+ * Class:     org_xerial_snappy_SnappyNative
+ * Method:    bitUnShuffle
+ * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
+ */
+JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
+  (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
+{
+	char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
+	char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
+	if(in == 0 || out == 0) {
+		// out of memory
+		if(in != 0) {
+			env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
+		}
+		if(out != 0) {
+			env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
+		}
+		throw_exception(env, self, 4);
+		return 0;
+	}
+
+        int64_t processedBytes = bshuf_bitunshuffle(
+                        in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
+
+	env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
+	env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
+
+	return (jint) processedBytes;
+}
+

src/main/java/org/xerial/snappy/BitShuffleNative.h

@@ -0,0 +1,29 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_xerial_snappy_BitShuffleNative */
+
+#ifndef _Included_org_xerial_snappy_BitShuffleNative
+#define _Included_org_xerial_snappy_BitShuffleNative
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_xerial_snappy_BitShuffleNative
+ * Method:    bitShuffle
+ * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
+ */
+JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
+  (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
+
+/*
+ * Class:     org_xerial_snappy_BitShuffleNative
+ * Method:    bitUnShuffle
+ * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
+ */
+JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
+  (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

src/main/java/org/xerial/snappy/BitShuffleNative.java

@@ -0,0 +1,53 @@
+/*--------------------------------------------------------------------------
+ *  Copyright 2011 Taro L. Saito
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *--------------------------------------------------------------------------*/
+//--------------------------------------
+// snappy-java Project
+//
+// BitShuffleNative.java
+// Since: 2016/03/31
+//
+// $URL$
+// $Author$
+//--------------------------------------
+package org.xerial.snappy;
+
+import java.io.IOException;
+
+/**
+ * JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
+ * defined in BitShuffleNative.h (genereted by javah) and BitShuffleNative.cpp
+ * <p/>
+ * <p>
+ * <b> DO NOT USE THIS CLASS since the direct use of this class might break the
+ * native library code loading in {@link SnappyLoader}. </b>
+ * </p>
+ *
+ * @author leo
+ */
+public class BitShuffleNative
+{
+
+    // ------------------------------------------------------------------------
+    // Bit-shuffling routines to improve compression of typed binary data.
+    // A quick benchmark result can be found in a gist below;
+    // https://gist.github.com/maropu/01103215df34b317a7a7
+    // ------------------------------------------------------------------------
+    public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
+            throws IOException;
+
+    public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
+            throws IOException;
+}

src/main/java/org/xerial/snappy/Snappy.java

@@ -44,7 +44,7 @@ public class Snappy
 {
     static {
         try {
-            impl = SnappyLoader.load();
+            impl = SnappyLoader.loadSnappyApi();
         }
         catch (Exception e) {
             throw new ExceptionInInitializerError(e);
@@ -66,7 +66,7 @@ public class Snappy
     public static void cleanUp()
     {
         SnappyLoader.cleanUpExtractedNativeLib();
-        SnappyLoader.setApi(null);
+        SnappyLoader.setSnappyApi(null);
     }
 
     /**
@@ -295,7 +295,7 @@ public class Snappy
                     versionData.load(in);
                     version = versionData.getProperty("version", version);
                     if (version.equals("unknown")) {
-                        version = versionData.getProperty("VERSION", version);
+                        version = versionData.getProperty("SNAPPY_VERSION", version);
                     }
                     version = version.trim().replaceAll("[^0-9\\.]", "");
                 }

src/main/java/org/xerial/snappy/SnappyBundleActivator.java

@@ -56,13 +56,13 @@ public class SnappyBundleActivator
             library = library.replace(".dylib", ".jnilib");
         }
         System.loadLibrary(library);
-        SnappyLoader.setApi(new SnappyNative());
+        SnappyLoader.setSnappyApi(new SnappyNative());
     }
 
     public void stop(BundleContext context)
             throws Exception
     {
-        SnappyLoader.setApi(null);
+        SnappyLoader.setSnappyApi(null);
         SnappyLoader.cleanUpExtractedNativeLib();
     }
 }

src/main/java/org/xerial/snappy/SnappyLoader.java

@@ -79,8 +79,10 @@ public class SnappyLoader
     public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib";
     public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility
 
-    private static volatile boolean isLoaded = false;
+    private static boolean isLoaded = false;
-    private static volatile SnappyNative api = null;
+
+    private static volatile SnappyNative snappyApi = null;
+    private static volatile BitShuffleNative bitshuffleApi = null;
 
     private static File nativeLibFile = null;
 
@@ -91,17 +93,19 @@ public class SnappyLoader
             if (!deleted) {
                 // Deleting native lib has failed, but it's not serious so simply ignore it here
             }
+            snappyApi = null;
+            bitshuffleApi = null;
         }
     }
 
     /**
-     * Set the api instance.
+     * Set the `snappyApi` instance.
      *
      * @param nativeCode
      */
-    static synchronized void setApi(SnappyNative nativeCode)
+    static synchronized void setSnappyApi(SnappyNative nativeCode)
     {
-        api = nativeCode;
+        snappyApi = nativeCode;
     }
 
     /**
@@ -142,42 +146,49 @@ public class SnappyLoader
         loadSnappySystemProperties();
     }
 
-    static synchronized SnappyNative load()
+    static synchronized SnappyNative loadSnappyApi()
     {
-        if (api != null) {
+        if (snappyApi != null) {
-            return api;
+            return snappyApi;
         }
-
-        try {
         loadNativeLibrary();
-
+        setSnappyApi(new SnappyNative());
-            setApi(new SnappyNative());
+        return snappyApi;
-            isLoaded = true;
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-            throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
     }
 
-        return api;
+    static synchronized BitShuffleNative loadBitShuffleApi()
+    {
+        if (bitshuffleApi != null) {
+            return bitshuffleApi;
+        }
+        loadNativeLibrary();
+        bitshuffleApi = new BitShuffleNative();
+        return bitshuffleApi;
     }
 
     /**
      * Load a native library of snappy-java
      */
-    private static void loadNativeLibrary()
+    private synchronized static void loadNativeLibrary()
     {
-
+        if (!isLoaded) {
+            try {
                 nativeLibFile = findNativeLibrary();
                 if (nativeLibFile != null) {
                     // Load extracted or specified snappyjava native library.
                     System.load(nativeLibFile.getAbsolutePath());
-        }
+                } else {
-        else {
                     // Load preinstalled snappyjava (in the path -Djava.library.path)
                     System.loadLibrary("snappyjava");
                 }
             }
+            catch (Exception e) {
+                e.printStackTrace();
+                throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
+            }
+            isLoaded = true;
+        }
+    }
 
     private static boolean contentsEquals(InputStream in1, InputStream in2)
             throws IOException
@@ -372,7 +383,7 @@ public class SnappyLoader
                 versionData.load(versionFile.openStream());
                 version = versionData.getProperty("version", version);
                 if (version.equals("unknown")) {
-                    version = versionData.getProperty("VERSION", version);
+                    version = versionData.getProperty("SNAPPY_VERSION", version);
                 }
                 version = version.trim().replaceAll("[^0-9M\\.]", "");
             }

src/main/java/org/xerial/snappy/SnappyNative.cpp

@@ -18,7 +18,7 @@
 #include <snappy.h>
 #include "SnappyNative.h"
 
-void throw_exception(JNIEnv *env, jobject self, int errorCode)
+inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
 {
 	jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
 	if(c==0)
@@ -29,11 +29,11 @@ void throw_exception(JNIEnv *env, jobject self, int errorCode)
 	env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
 }
 
-
 JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersion
   (JNIEnv * env, jobject self)
 {
-	return env->NewStringUTF("1.1.0");
+	// TODO: Do we need to read this library version from resources/org/xerial/snappy/VERSION?
+	return env->NewStringUTF("1.1.3");
 }
 
 JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
@@ -41,9 +41,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
      size_t compressedLength;
      snappy::RawCompress((char*) srcAddr, (size_t) length, (char*) destAddr, &compressedLength);
      return (jlong) compressedLength;
-  }
+}
-
-
 
 JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
   (JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
@@ -58,9 +56,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
  	}
 
  	return (jlong) uncompressedLength;
-  }
+}
-
-
 
 /*
  * Class:     org_xerial_snappy_Snappy
@@ -82,7 +78,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_ni
 	return (jint) compressedLength;
 }
 
-
 JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_lang_Object_2IILjava_lang_Object_2I
   (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLen, jobject output, jint outputOffset)
 {
@@ -141,7 +136,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_
 	return (jint) uncompressedLength;
 }
 
-
 /*
  * Class:     org_xerial_snappy_Snappy
  * Method:    uncompress
@@ -168,8 +162,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_
 	return (jint) decompressedLength;
 }
 
-
-
 /*
  * Class:     org_xerial_snappy_Snappy
  * Method:    maxCompressedLength
@@ -240,8 +232,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__
 	}
 
 	return (jint) result;
-  }
+}
-
 
 JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II
   (JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
@@ -306,4 +297,3 @@ JNIEXPORT void JNICALL Java_org_xerial_snappy_SnappyNative_arrayCopy
 	env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
 }
 
-

src/main/resources/org/xerial/snappy/VERSION

@@ -1 +1,2 @@
-VERSION=1.1.3
+SNAPPY_VERSION=1.1.3
+BITSHUFFLE_VERSION=0.2.2

src/test/java/org/xerial/snappy/BitShuffleTest.java

@@ -0,0 +1,82 @@
+/*--------------------------------------------------------------------------
+ *  Copyright 2011 Taro L. Saito
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *--------------------------------------------------------------------------*/
+//--------------------------------------
+// snappy-java Project
+//
+// BitShuffleTest.java
+// Since: 2016/03/31
+//
+// $URL$
+// $Author$
+//--------------------------------------
+package org.xerial.snappy;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class BitShuffleTest {
+
+    @Test
+    public void bitShuffleLongArray()
+            throws Exception
+    {
+        long[] data = new long[] {2, 3, 15, 4234, 43251531412342342L, 23423422342L};
+        byte[] shuffledData = BitShuffle.bitShuffle(data);
+        long[] result = BitShuffle.bitUnShuffleLongArray(shuffledData);
+        assertArrayEquals(data, result);
+    }
+
+    @Test
+    public void bitShuffleShortArray()
+            throws Exception
+    {
+        short[] data = new short[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1};
+        byte[] shuffledData = BitShuffle.bitShuffle(data);
+        short[] result = BitShuffle.bitUnShuffleShortArray(shuffledData);
+        assertArrayEquals(data, result);
+    }
+
+    @Test
+    public void bitShuffleIntArray()
+            throws Exception
+    {
+        int[] data = new int[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43};
+        byte[] shuffledData = BitShuffle.bitShuffle(data);
+        int[] result = BitShuffle.bitUnShuffleIntArray(shuffledData);
+        assertArrayEquals(data, result);
+    }
+
+    @Test
+    public void bitShuffleFloatArray()
+            throws Exception
+    {
+        float[] data = new float[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
+        byte[] shuffledData = BitShuffle.bitShuffle(data);
+        float[] result = BitShuffle.bitUnShuffleFloatArray(shuffledData);
+        assertArrayEquals(data, result, 0.0000001f);
+    }
+
+    @Test
+    public void bitShuffleDoubleArray()
+            throws Exception
+    {
+        double[] data = new double[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
+        byte[] shuffledData = BitShuffle.bitShuffle(data);
+        double[] result = BitShuffle.bitUnShuffleDoubleArray(shuffledData);
+        assertArrayEquals(data, result, 0.0000001f);
+    }
+}

src/test/java/org/xerial/snappy/SnappyLoaderTest.java

@@ -117,7 +117,7 @@ public class SnappyLoaderTest
     public void load()
             throws Exception
     {
-        SnappyLoader.load();
+        SnappyLoader.loadSnappyApi();
         _logger.debug(Snappy.maxCompressedLength(1024));
     }