diff --git a/Makefile b/Makefile index 364c95c..d26aec0 100644 --- a/Makefile +++ b/Makefile @@ -7,17 +7,50 @@ SBT:=./sbt all: snappy SNAPPY_OUT:=$(TARGET)/$(snappy)-$(os_arch) -SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(VERSION).tar.gz +SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc -SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(VERSION) +SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION) SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC)) -SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) SnappyNative.o) - -SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy SNAPPY_GIT_REV:=2b9152d9c5bed71dffb7f7f6c7a3ec48b058ff2d # 1.1.3 with autogen.sh fix +SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log +SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log -CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) +BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz +BITSHUFFLE_C:=bitshuffle_core.c iochain.c +BITSHUFFLE_SRC_DIR:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION)/src +BITSHUFFLE_SRC:=$(addprefix $(BITSHUFFLE_SRC_DIR)/,$(BITSHUFFLE_C)) +BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log + +$(BITSHUFFLE_ARCHIVE): + @mkdir -p $(@D) + curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz + +$(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE) + $(TAR) xvfz $< -C $(TARGET) + touch $@ + +$(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED) + +$(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(CXXFLAGS_BITSHUFFLE) -c $< -o $@ + +SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o) + +CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(BITSHUFFLE_SRC_DIR) + +ifndef CXXFLAGS_BITSHUFFLE + ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64) + # SSE2 is supported in all the x86_64 platforms and AVX2 is only supported + # in the small part of them. gcc in linux/x86_64 typically enables SSE2 by default though, + # we explicitly set flags below to make this precondition clearer. + CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -msse2 + else + # Undefined macros to generate a platform-independent binary + CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -U__SSE2__ + endif +endif ifeq ($(OS_NAME),SunOS) TAR:= gtar @@ -27,7 +60,12 @@ endif $(SNAPPY_ARCHIVE): @mkdir -p $(@D) - curl -L -o$@ https://github.com/google/snappy/releases/download/$(VERSION)/snappy-$(VERSION).tar.gz + curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz + +$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE) + $(TAR) xvfz $< -C $(TARGET) + touch $@ + cd $(SNAPPY_SRC_DIR) && ./configure $(SNAPPY_GIT_UNPACKED): rm -rf $(SNAPPY_SRC_DIR) @@ -37,31 +75,41 @@ $(SNAPPY_GIT_UNPACKED): cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure touch $@ -jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h +jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h -$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/snappy/SnappyNative.java +$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java @mkdir -p $(TARGET)/jni-classes $(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $< $(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class $(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative +$(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java + @mkdir -p $(TARGET)/jni-classes + $(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $< + +$(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class + $(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.BitShuffleNative + $(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED) -$(SNAPPY_OUT)/%.o : $(SNAPPY_SRC_DIR)/%.cc - @mkdir -p $(@D) - $(CXX) $(CXXFLAGS) -c $< -o $@ - -$(SNAPPY_OUT)/SnappyNative.o : $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h +$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc @mkdir -p $(@D) $(CXX) $(CXXFLAGS) -c $< -o $@ +$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) -c $< -o $@ + +$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) -c $< -o $@ $(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ) - $(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS) + $(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS) $(STRIP) $@ -clean-native: +clean-native: rm -rf $(SNAPPY_OUT) clean: @@ -76,7 +124,7 @@ snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s native: $(SNAPPY_GIT_UNPACKED) $(NATIVE_DLL) snappy: native $(TARGET)/$(snappy-jar-version).jar -$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME) +$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME) @mkdir -p $(@D) cp $< $@ @mkdir -p $(NATIVE_TARGET_DIR) @@ -85,20 +133,20 @@ $(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME) package: $(TARGET)/$(snappy-jar-version).jar -$(TARGET)/$(snappy-jar-version).jar: - $(SBT) package +$(TARGET)/$(snappy-jar-version).jar: + $(SBT) package test: $(NATIVE_DLL) $(SBT) test -win32: +win32: $(MAKE) native CROSS_PREFIX=i686-w64-mingw32- OS_NAME=Windows OS_ARCH=x86 # for cross-compilation on Ubuntu, install the g++-mingw-w64-x86-64 package win64: $(MAKE) native CROSS_PREFIX=x86_64-w64-mingw32- OS_NAME=Windows OS_ARCH=x86_64 -mac32: +mac32: $(MAKE) native OS_NAME=Mac OS_ARCH=x86 linux32: @@ -133,17 +181,16 @@ install-m2: googlecode-upload: googlecode-lib-upload googlecode-src-upload -googlecode-lib-upload: $(TARGET)/snappy-java-$(VERSION)-lib.upload -googlecode-src-upload: $(TARGET)/snappy-java-$(VERSION)-src.upload +googlecode-lib-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload +googlecode-src-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload GOOGLECODE_USER:=leo@xerial.org -$(TARGET)/snappy-java-$(VERSION)-lib.upload: - ./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).jar +$(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload: + ./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).jar touch $@ -$(TARGET)/snappy-java-$(VERSION)-src.upload: - ./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).tar.gz +$(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload: + ./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).tar.gz touch $@ - diff --git a/Makefile.common b/Makefile.common index d198b1f..cc605f1 100755 --- a/Makefile.common +++ b/Makefile.common @@ -8,13 +8,13 @@ endif JAVA := "$$JAVA_HOME/bin/java" -JAVAC := "$$JAVA_HOME/bin/javac" +JAVAC := "$$JAVA_HOME/bin/javac" JAVAH := "$$JAVA_HOME/bin/javah" OSINFO_CLASS := org.xerial.snappy.OSInfo OSINFO_PROG := lib/org/xerial/snappy/OSInfo.class -## building OSInfo.java +## building OSInfo.java #$(info compiling OSInfo.java) #$(shell mkdir -p lib) #$(shell $(JAVAC) src/main/java/org/xerial/snappy/OSInfo.java -d lib) @@ -31,7 +31,7 @@ else sep := : endif -snappy := snappy-$(VERSION) +snappy := snappy-$(SNAPPY_VERSION) jni_md := $(shell find -L "$(JAVA_HOME)" -name jni_md.h | head -1) @@ -70,7 +70,7 @@ Default_STRIP := $(CROSS_PREFIX)strip Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden Default_LINKFLAGS := -shared -static Default_LIBNAME := libsnappyjava.so -Default_SNAPPY_FLAGS := +Default_SNAPPY_FLAGS := Linux-x86_CXX := $(CROSS_PREFIX)g++ Linux-x86_STRIP := $(CROSS_PREFIX)strip @@ -81,18 +81,18 @@ else endif Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-x86_LIBNAME := libsnappyjava.so -Linux-x86_SNAPPY_FLAGS:= +Linux-x86_SNAPPY_FLAGS:= -Linux-x86_64_CXX := $(CROSS_PREFIX)g++ +Linux-x86_64_CXX := $(CROSS_PREFIX)g++ Linux-x86_64_STRIP := $(CROSS_PREFIX)strip ifeq ($(IBM_JDK_7),) - Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 else - Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 endif Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-x86_64_LIBNAME := libsnappyjava.so -Linux-x86_64_SNAPPY_FLAGS := +Linux-x86_64_SNAPPY_FLAGS := Linux-ppc_CXX := g++ Linux-ppc_STRIP := strip @@ -108,7 +108,7 @@ Linux-ppc_SNAPPY_FLAGS := Linux-ppc64_CXX := g++ Linux-ppc64_STRIP := strip ifeq ($(IBM_JDK_7),) - Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 + Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 else Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 endif @@ -160,19 +160,19 @@ Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-s390x_LIBNAME := libsnappyjava.so Linux-s390x_SNAPPY_FLAGS := -SunOS-x86_CXX := g++ +SunOS-x86_CXX := g++ SunOS-x86_STRIP := strip SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden SunOS-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++ SunOS-x86_LIBNAME := libsnappyjava.so -SunOS-x86_SNAPPY_FLAGS := +SunOS-x86_SNAPPY_FLAGS := -SunOS-sparc_CXX := g++ +SunOS-sparc_CXX := g++ SunOS-sparc_STRIP := strip SunOS-sparc_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden SunOS-sparc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ SunOS-sparc_LIBNAME := libsnappyjava.so -SunOS-sparc_SNAPPY_FLAGS := +SunOS-sparc_SNAPPY_FLAGS := SunOS-x86_64_CXX := g++ SunOS-x86_64_STRIP := strip @@ -191,14 +191,14 @@ Linux-arm_STRIP := $(CROSS_PREFIX)strip Linux-arm_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp Linux-arm_LINKFLAGS := -shared -static-libgcc Linux-arm_LIBNAME := libsnappyjava.so -Linux-arm_SNAPPY_FLAGS:= +Linux-arm_SNAPPY_FLAGS:= Linux-armhf_CXX := $(CROSS_PREFIX)g++ Linux-armhf_STRIP := $(CROSS_PREFIX)strip Linux-armhf_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard Linux-armhf_LINKFLAGS := -shared -static-libgcc Linux-armhf_LIBNAME := libsnappyjava.so -Linux-armhf_SNAPPY_FLAGS:= +Linux-armhf_SNAPPY_FLAGS:= Linux-aarch64_CXX := $(CROSS_PREFIX)g++ Linux-aarch64_STRIP := $(CROSS_PREFIX)strip @@ -214,47 +214,46 @@ Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc Mac-x86_LIBNAME := libsnappyjava.jnilib Mac-x86_SNAPPY_FLAGS := -Mac-x86_64_CXX := g++ -arch $(OS_ARCH) +Mac-x86_64_CXX := g++ -arch $(OS_ARCH) Mac-x86_64_STRIP := strip -x Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden -Mac-x86_64_LINKFLAGS := -dynamiclib -Mac-x86_64_LIBNAME := libsnappyjava.jnilib -Mac-x86_64_SNAPPY_FLAGS := +Mac-x86_64_LINKFLAGS := -dynamiclib +Mac-x86_64_LIBNAME := libsnappyjava.jnilib +Mac-x86_64_SNAPPY_FLAGS := FreeBSD-x86_64_CXX := $(CROSS_PREFIX)g++ FreeBSD-x86_64_STRIP := $(CROSS_PREFIX)strip FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden FreeBSD-x86_64_LINKFLAGS := -shared -static-libgcc FreeBSD-x86_64_LIBNAME := libsnappyjava.so -FreeBSD-x86_64_SNAPPY_FLAGS := +FreeBSD-x86_64_SNAPPY_FLAGS := Windows-x86_CXX := $(CROSS_PREFIX)g++ Windows-x86_STRIP := $(CROSS_PREFIX)strip -Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 +Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 Windows-x86_LINKFLAGS := -Wl,--kill-at -shared -static Windows-x86_LIBNAME := snappyjava.dll -Windows-x86_SNAPPY_FLAGS := +Windows-x86_SNAPPY_FLAGS := Windows-x86_64_CXX := $(CROSS_PREFIX)g++ Windows-x86_64_STRIP := $(CROSS_PREFIX)strip -Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 -Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static +Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 +Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static Windows-x86_64_LIBNAME := snappyjava.dll -Windows-x86_64_SNAPPY_FLAGS := +Windows-x86_64_SNAPPY_FLAGS := CXX := $($(os_arch)_CXX) STRIP := $($(os_arch)_STRIP) -CXXFLAGS := $($(os_arch)_CXXFLAGS) -LINKFLAGS := $($(os_arch)_LINKFLAGS) +CXXFLAGS := $($(os_arch)_CXXFLAGS) +LINKFLAGS := $($(os_arch)_LINKFLAGS) LIBNAME := $($(os_arch)_LIBNAME) SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS) -CXXFLAGS := $(CXXFLAGS) -Ilib/include +CXXFLAGS := $(CXXFLAGS) -Ilib/include ifneq ($(jni_include),) CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)" endif - diff --git a/README.md b/README.md index c4950a6..378739d 100755 --- a/README.md +++ b/README.md @@ -143,6 +143,12 @@ Older snapshots of snappy contain a buggy config.h.in that does not work properl $ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x $ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1 +## Activating SSE2/AVX2 instructions in BitShuffle + +The most of the native libraries that snappy-java contains disable SSE2/AVX2 instructions in terms of portability (SSE2 is enabled only in Linux/x86_64 platforms). To enable AVX2 instructions, you need to compile as follows: + + $ make CXXFLAGS_BITSHUFFLE=-mavx2 # -msse2 for SSE2 instructions + ## Cross-compiling for other platforms The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM: diff --git a/src/main/java/org/xerial/snappy/BitShuffle.java b/src/main/java/org/xerial/snappy/BitShuffle.java new file mode 100644 index 0000000..7cca5be --- /dev/null +++ b/src/main/java/org/xerial/snappy/BitShuffle.java @@ -0,0 +1,184 @@ +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// snappy-java Project +// +// BitShuffle.java +// Since: 2016/03/31 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import java.io.IOException; + +public class BitShuffle +{ + static { + try { + impl = SnappyLoader.loadBitShuffleApi(); + } + catch (Exception e) { + throw new ExceptionInInitializerError(e); + } + } + + /** + * An instance of BitShuffleNative + */ + private static BitShuffleNative impl; + + /** + * Apply a bit-shuffling filter into the input short array. + * + * @param input + * @return bit-shuffled byte array + * @throws IOException + */ + public static byte[] bitShuffle(short[] input) throws IOException { + byte[] output = new byte[input.length * 2]; + int numProcessed = impl.bitShuffle(input, 0, 2, input.length * 2, output, 0); + assert(numProcessed == input.length * 2); + return output; + } + + /** + * Apply a bit-shuffling filter into the input int array. + * + * @param input + * @return bit-shuffled byte array + * @throws IOException + */ + public static byte[] bitShuffle(int[] input) throws IOException { + byte[] output = new byte[input.length * 4]; + int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0); + assert(numProcessed == input.length * 4); + return output; + } + + /** + * Apply a bit-shuffling filter into the input long array. + * + * @param input + * @return bit-shuffled byte array + * @throws IOException + */ + public static byte[] bitShuffle(long[] input) throws IOException { + byte[] output = new byte[input.length * 8]; + int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0); + assert(numProcessed == input.length * 8); + return output; + } + + /** + * Apply a bit-shuffling filter into the input float array. + * + * @param input + * @return bit-shuffled byte array + * @throws IOException + */ + public static byte[] bitShuffle(float[] input) throws IOException { + byte[] output = new byte[input.length * 4]; + int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0); + assert(numProcessed == input.length * 4); + return output; + } + + /** + * Apply a bit-shuffling filter into the input double array. + * + * @param input + * @return bit-shuffled byte array + * @throws IOException + */ + public static byte[] bitShuffle(double[] input) throws IOException { + byte[] output = new byte[input.length * 8]; + int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0); + assert(numProcessed == input.length * 8); + return output; + } + + /** + * Convert the input bit-shuffled byte array into an original short array. + * + * @param input + * @return a short array + * @throws IOException + */ + public static short[] bitUnShuffleShortArray(byte[] input) throws IOException { + short[] output = new short[input.length / 2]; + int numProcessed = impl.bitUnShuffle(input, 0, 2, input.length, output, 0); + assert(numProcessed == input.length); + return output; + } + + /** + * Convert the input bit-shuffled byte array into an original int array. + * + * @param input + * @return an int array + * @throws IOException + */ + public static int[] bitUnShuffleIntArray(byte[] input) throws IOException { + int[] output = new int[input.length / 4]; + int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0); + assert(numProcessed == input.length); + return output; + } + + /** + * Convert the input bit-shuffled byte array into an original long array. + * + * @param input + * @return a long array + * @throws IOException + */ + public static long[] bitUnShuffleLongArray(byte[] input) throws IOException { + long[] output = new long[input.length / 8]; + int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0); + assert(numProcessed == input.length); + return output; + } + + /** + * Convert the input bit-shuffled byte array into an original float array. + * + * @param input + * @return an float array + * @throws IOException + */ + public static float[] bitUnShuffleFloatArray(byte[] input) throws IOException { + float[] output = new float[input.length / 4]; + int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0); + assert(numProcessed == input.length); + return output; + } + + /** + * Convert the input bit-shuffled byte array into an original double array. + * + * @param input + * @return a double array + * @throws IOException + */ + public static double[] bitUnShuffleDoubleArray(byte[] input) throws IOException { + double[] output = new double[input.length / 8]; + int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0); + assert(numProcessed == input.length); + return output; + } +} diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.cpp b/src/main/java/org/xerial/snappy/BitShuffleNative.cpp new file mode 100755 index 0000000..eb79a16 --- /dev/null +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.cpp @@ -0,0 +1,91 @@ +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +#include +#include "BitShuffleNative.h" + +inline void throw_exception(JNIEnv *env, jobject self, int errorCode) +{ + jclass c = env->FindClass("org/xerial/snappy/SnappyNative"); + if(c==0) + return; + jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V"); + if(mth_throwex == 0) + return; + env->CallVoidMethod(self, mth_throwex, (jint) errorCode); +} + +/* + * Class: org_xerial_snappy_SnappyNative + * Method: bitShuffle + * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle + (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset) +{ + char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0); + char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0); + if(in == 0 || out == 0) { + // out of memory + if(in != 0) { + env->ReleasePrimitiveArrayCritical((jarray) input, in, 0); + } + if(out != 0) { + env->ReleasePrimitiveArrayCritical((jarray) output, out, 0); + } + throw_exception(env, self, 4); + return 0; + } + + int64_t processedBytes = bshuf_bitshuffle( + in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0); + + env->ReleasePrimitiveArrayCritical((jarray) input, in, 0); + env->ReleasePrimitiveArrayCritical((jarray) output, out, 0); + + return (jint) processedBytes; +} + +/* + * Class: org_xerial_snappy_SnappyNative + * Method: bitUnShuffle + * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle + (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset) +{ + char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0); + char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0); + if(in == 0 || out == 0) { + // out of memory + if(in != 0) { + env->ReleasePrimitiveArrayCritical((jarray) input, in, 0); + } + if(out != 0) { + env->ReleasePrimitiveArrayCritical((jarray) output, out, 0); + } + throw_exception(env, self, 4); + return 0; + } + + int64_t processedBytes = bshuf_bitunshuffle( + in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0); + + env->ReleasePrimitiveArrayCritical((jarray) input, in, 0); + env->ReleasePrimitiveArrayCritical((jarray) output, out, 0); + + return (jint) processedBytes; +} + diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.h b/src/main/java/org/xerial/snappy/BitShuffleNative.h new file mode 100644 index 0000000..3f71339 --- /dev/null +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_xerial_snappy_BitShuffleNative */ + +#ifndef _Included_org_xerial_snappy_BitShuffleNative +#define _Included_org_xerial_snappy_BitShuffleNative +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitShuffle + * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle + (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); + +/* + * Class: org_xerial_snappy_BitShuffleNative + * Method: bitUnShuffle + * Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I + */ +JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle + (JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/java/org/xerial/snappy/BitShuffleNative.java b/src/main/java/org/xerial/snappy/BitShuffleNative.java new file mode 100644 index 0000000..57a61d4 --- /dev/null +++ b/src/main/java/org/xerial/snappy/BitShuffleNative.java @@ -0,0 +1,53 @@ +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// snappy-java Project +// +// BitShuffleNative.java +// Since: 2016/03/31 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import java.io.IOException; + +/** + * JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is + * defined in BitShuffleNative.h (genereted by javah) and BitShuffleNative.cpp + *

+ *

+ * DO NOT USE THIS CLASS since the direct use of this class might break the + * native library code loading in {@link SnappyLoader}. + *

+ * + * @author leo + */ +public class BitShuffleNative +{ + + // ------------------------------------------------------------------------ + // Bit-shuffling routines to improve compression of typed binary data. + // A quick benchmark result can be found in a gist below; + // https://gist.github.com/maropu/01103215df34b317a7a7 + // ------------------------------------------------------------------------ + public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) + throws IOException; + + public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset) + throws IOException; +} diff --git a/src/main/java/org/xerial/snappy/Snappy.java b/src/main/java/org/xerial/snappy/Snappy.java index dc81f7c..31d0d2e 100755 --- a/src/main/java/org/xerial/snappy/Snappy.java +++ b/src/main/java/org/xerial/snappy/Snappy.java @@ -44,7 +44,7 @@ public class Snappy { static { try { - impl = SnappyLoader.load(); + impl = SnappyLoader.loadSnappyApi(); } catch (Exception e) { throw new ExceptionInInitializerError(e); @@ -66,7 +66,7 @@ public class Snappy public static void cleanUp() { SnappyLoader.cleanUpExtractedNativeLib(); - SnappyLoader.setApi(null); + SnappyLoader.setSnappyApi(null); } /** @@ -295,7 +295,7 @@ public class Snappy versionData.load(in); version = versionData.getProperty("version", version); if (version.equals("unknown")) { - version = versionData.getProperty("VERSION", version); + version = versionData.getProperty("SNAPPY_VERSION", version); } version = version.trim().replaceAll("[^0-9\\.]", ""); } diff --git a/src/main/java/org/xerial/snappy/SnappyBundleActivator.java b/src/main/java/org/xerial/snappy/SnappyBundleActivator.java index 9dd834d..dec838c 100755 --- a/src/main/java/org/xerial/snappy/SnappyBundleActivator.java +++ b/src/main/java/org/xerial/snappy/SnappyBundleActivator.java @@ -56,13 +56,13 @@ public class SnappyBundleActivator library = library.replace(".dylib", ".jnilib"); } System.loadLibrary(library); - SnappyLoader.setApi(new SnappyNative()); + SnappyLoader.setSnappyApi(new SnappyNative()); } public void stop(BundleContext context) throws Exception { - SnappyLoader.setApi(null); + SnappyLoader.setSnappyApi(null); SnappyLoader.cleanUpExtractedNativeLib(); } } diff --git a/src/main/java/org/xerial/snappy/SnappyLoader.java b/src/main/java/org/xerial/snappy/SnappyLoader.java index 11f40a4..27855af 100755 --- a/src/main/java/org/xerial/snappy/SnappyLoader.java +++ b/src/main/java/org/xerial/snappy/SnappyLoader.java @@ -79,8 +79,10 @@ public class SnappyLoader public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib"; public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility - private static volatile boolean isLoaded = false; - private static volatile SnappyNative api = null; + private static boolean isLoaded = false; + + private static volatile SnappyNative snappyApi = null; + private static volatile BitShuffleNative bitshuffleApi = null; private static File nativeLibFile = null; @@ -91,17 +93,19 @@ public class SnappyLoader if (!deleted) { // Deleting native lib has failed, but it's not serious so simply ignore it here } + snappyApi = null; + bitshuffleApi = null; } } /** - * Set the api instance. + * Set the `snappyApi` instance. * * @param nativeCode */ - static synchronized void setApi(SnappyNative nativeCode) + static synchronized void setSnappyApi(SnappyNative nativeCode) { - api = nativeCode; + snappyApi = nativeCode; } /** @@ -142,40 +146,47 @@ public class SnappyLoader loadSnappySystemProperties(); } - static synchronized SnappyNative load() + static synchronized SnappyNative loadSnappyApi() { - if (api != null) { - return api; + if (snappyApi != null) { + return snappyApi; } + loadNativeLibrary(); + setSnappyApi(new SnappyNative()); + return snappyApi; + } - try { - loadNativeLibrary(); - - setApi(new SnappyNative()); - isLoaded = true; + static synchronized BitShuffleNative loadBitShuffleApi() + { + if (bitshuffleApi != null) { + return bitshuffleApi; } - catch (Exception e) { - e.printStackTrace(); - throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage()); - } - - return api; + loadNativeLibrary(); + bitshuffleApi = new BitShuffleNative(); + return bitshuffleApi; } /** * Load a native library of snappy-java */ - private static void loadNativeLibrary() + private synchronized static void loadNativeLibrary() { - - nativeLibFile = findNativeLibrary(); - if (nativeLibFile != null) { - // Load extracted or specified snappyjava native library. - System.load(nativeLibFile.getAbsolutePath()); - } - else { - // Load preinstalled snappyjava (in the path -Djava.library.path) - System.loadLibrary("snappyjava"); + if (!isLoaded) { + try { + nativeLibFile = findNativeLibrary(); + if (nativeLibFile != null) { + // Load extracted or specified snappyjava native library. + System.load(nativeLibFile.getAbsolutePath()); + } else { + // Load preinstalled snappyjava (in the path -Djava.library.path) + System.loadLibrary("snappyjava"); + } + } + catch (Exception e) { + e.printStackTrace(); + throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage()); + } + isLoaded = true; } } @@ -372,7 +383,7 @@ public class SnappyLoader versionData.load(versionFile.openStream()); version = versionData.getProperty("version", version); if (version.equals("unknown")) { - version = versionData.getProperty("VERSION", version); + version = versionData.getProperty("SNAPPY_VERSION", version); } version = version.trim().replaceAll("[^0-9M\\.]", ""); } diff --git a/src/main/java/org/xerial/snappy/SnappyNative.cpp b/src/main/java/org/xerial/snappy/SnappyNative.cpp index d635880..aacd629 100755 --- a/src/main/java/org/xerial/snappy/SnappyNative.cpp +++ b/src/main/java/org/xerial/snappy/SnappyNative.cpp @@ -18,22 +18,22 @@ #include #include "SnappyNative.h" -void throw_exception(JNIEnv *env, jobject self, int errorCode) +inline void throw_exception(JNIEnv *env, jobject self, int errorCode) { jclass c = env->FindClass("org/xerial/snappy/SnappyNative"); if(c==0) return; - jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V"); - if(mth_throwex == 0) - return; - env->CallVoidMethod(self, mth_throwex, (jint) errorCode); + jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V"); + if(mth_throwex == 0) + return; + env->CallVoidMethod(self, mth_throwex, (jint) errorCode); } - JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersion (JNIEnv * env, jobject self) { - return env->NewStringUTF("1.1.0"); + // TODO: Do we need to read this library version from resources/org/xerial/snappy/VERSION? + return env->NewStringUTF("1.1.3"); } JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ @@ -41,9 +41,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ size_t compressedLength; snappy::RawCompress((char*) srcAddr, (size_t) length, (char*) destAddr, &compressedLength); return (jlong) compressedLength; - } - - +} JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ (JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) { @@ -58,9 +56,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ } return (jlong) uncompressedLength; - } - - +} /* * Class: org_xerial_snappy_Snappy @@ -82,7 +78,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_ni return (jint) compressedLength; } - JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_lang_Object_2IILjava_lang_Object_2I (JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLen, jobject output, jint outputOffset) { @@ -141,7 +136,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_ return (jint) uncompressedLength; } - /* * Class: org_xerial_snappy_Snappy * Method: uncompress @@ -168,8 +162,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_ return (jint) decompressedLength; } - - /* * Class: org_xerial_snappy_Snappy * Method: maxCompressedLength @@ -240,8 +232,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__ } return (jint) result; - } - +} JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II (JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen) @@ -306,4 +297,3 @@ JNIEXPORT void JNICALL Java_org_xerial_snappy_SnappyNative_arrayCopy env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0); } - diff --git a/src/main/java/org/xerial/snappy/SnappyNative.h b/src/main/java/org/xerial/snappy/SnappyNative.h old mode 100755 new mode 100644 diff --git a/src/main/resources/org/xerial/snappy/VERSION b/src/main/resources/org/xerial/snappy/VERSION index 5582f56..e9a5db2 100755 --- a/src/main/resources/org/xerial/snappy/VERSION +++ b/src/main/resources/org/xerial/snappy/VERSION @@ -1 +1,2 @@ -VERSION=1.1.3 +SNAPPY_VERSION=1.1.3 +BITSHUFFLE_VERSION=0.2.2 diff --git a/src/test/java/org/xerial/snappy/BitShuffleTest.java b/src/test/java/org/xerial/snappy/BitShuffleTest.java new file mode 100644 index 0000000..49f904d --- /dev/null +++ b/src/test/java/org/xerial/snappy/BitShuffleTest.java @@ -0,0 +1,82 @@ +/*-------------------------------------------------------------------------- + * Copyright 2011 Taro L. Saito + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------*/ +//-------------------------------------- +// snappy-java Project +// +// BitShuffleTest.java +// Since: 2016/03/31 +// +// $URL$ +// $Author$ +//-------------------------------------- +package org.xerial.snappy; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class BitShuffleTest { + + @Test + public void bitShuffleLongArray() + throws Exception + { + long[] data = new long[] {2, 3, 15, 4234, 43251531412342342L, 23423422342L}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + long[] result = BitShuffle.bitUnShuffleLongArray(shuffledData); + assertArrayEquals(data, result); + } + + @Test + public void bitShuffleShortArray() + throws Exception + { + short[] data = new short[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + short[] result = BitShuffle.bitUnShuffleShortArray(shuffledData); + assertArrayEquals(data, result); + } + + @Test + public void bitShuffleIntArray() + throws Exception + { + int[] data = new int[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + int[] result = BitShuffle.bitUnShuffleIntArray(shuffledData); + assertArrayEquals(data, result); + } + + @Test + public void bitShuffleFloatArray() + throws Exception + { + float[] data = new float[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + float[] result = BitShuffle.bitUnShuffleFloatArray(shuffledData); + assertArrayEquals(data, result, 0.0000001f); + } + + @Test + public void bitShuffleDoubleArray() + throws Exception + { + double[] data = new double[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + double[] result = BitShuffle.bitUnShuffleDoubleArray(shuffledData); + assertArrayEquals(data, result, 0.0000001f); + } +} diff --git a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java index 1cee9c5..2bc2148 100755 --- a/src/test/java/org/xerial/snappy/SnappyLoaderTest.java +++ b/src/test/java/org/xerial/snappy/SnappyLoaderTest.java @@ -117,7 +117,7 @@ public class SnappyLoaderTest public void load() throws Exception { - SnappyLoader.load(); + SnappyLoader.loadSnappyApi(); _logger.debug(Snappy.maxCompressedLength(1024)); }