tyler.durden
/
snappy-java
mirror of https://github.com/xerial/snappy-java.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #137 from maropu/SupportBitShuffle 

[WIP] Add bit-shuffling interfaces to improve LZ performance
This commit is contained in:
Taro L. Saito 2017-01-19 10:02:07 -08:00 committed by GitHub
parent 7352efda48 3a8e8c8f77
commit 1ef5cce1b5
15 changed files with 608 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
Makefile
View File

@ -7,17 +7,50 @@ SBT:=./sbt
all: snappy
SNAPPY_OUT:=$(TARGET)/$(snappy)-$(os_arch)
SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(VERSION).tar.gz
SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz
SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(VERSION)
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) SnappyNative.o)
SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
SNAPPY_GIT_REV:=2b9152d9c5bed71dffb7f7f6c7a3ec48b058ff2d # 1.1.3 with autogen.sh fix
SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR)
BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz
BITSHUFFLE_C:=bitshuffle_core.c iochain.c
BITSHUFFLE_SRC_DIR:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION)/src
BITSHUFFLE_SRC:=$(addprefix $(BITSHUFFLE_SRC_DIR)/,$(BITSHUFFLE_C))
BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log
$(BITSHUFFLE_ARCHIVE):
@mkdir -p $(@D)
curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz
$(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET)
touch $@
$(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED)
$(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) $(CXXFLAGS_BITSHUFFLE) -c $< -o $@
SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o)
CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(BITSHUFFLE_SRC_DIR)
ifndef CXXFLAGS_BITSHUFFLE
ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64)
# SSE2 is supported in all the x86_64 platforms and AVX2 is only supported
# in the small part of them. gcc in linux/x86_64 typically enables SSE2 by default though,
# we explicitly set flags below to make this precondition clearer.
CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -msse2
else
# Undefined macros to generate a platform-independent binary
CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -U__SSE2__
endif
endif
ifeq ($(OS_NAME),SunOS)
TAR:= gtar
@ -27,7 +60,12 @@ endif
$(SNAPPY_ARCHIVE):
@mkdir -p $(@D)
curl -L -o$@ https://github.com/google/snappy/releases/download/$(VERSION)/snappy-$(VERSION).tar.gz
curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz
$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET)
touch $@
cd $(SNAPPY_SRC_DIR) && ./configure
$(SNAPPY_GIT_UNPACKED):
rm -rf $(SNAPPY_SRC_DIR)
@ -37,31 +75,41 @@ $(SNAPPY_GIT_UNPACKED):
cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure
touch $@
jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h
jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/snappy/SnappyNative.java
$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java
@mkdir -p $(TARGET)/jni-classes
$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class
$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative
$(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java
@mkdir -p $(TARGET)/jni-classes
$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class
$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.BitShuffleNative
$(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
$(SNAPPY_OUT)/%.o : $(SNAPPY_SRC_DIR)/%.cc
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/SnappyNative.o : $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ)
$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
$(STRIP) $@
clean-native:
clean-native:
rm -rf $(SNAPPY_OUT)
clean:
@ -76,7 +124,7 @@ snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s
native: $(SNAPPY_GIT_UNPACKED) $(NATIVE_DLL)
snappy: native $(TARGET)/$(snappy-jar-version).jar
$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
@mkdir -p $(@D)
cp $< $@
@mkdir -p $(NATIVE_TARGET_DIR)
@ -85,20 +133,20 @@ $(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
package: $(TARGET)/$(snappy-jar-version).jar
$(TARGET)/$(snappy-jar-version).jar:
$(SBT) package
$(TARGET)/$(snappy-jar-version).jar:
$(SBT) package
test: $(NATIVE_DLL)
$(SBT) test
win32:
win32:
$(MAKE) native CROSS_PREFIX=i686-w64-mingw32- OS_NAME=Windows OS_ARCH=x86
# for cross-compilation on Ubuntu, install the g++-mingw-w64-x86-64 package
win64:
$(MAKE) native CROSS_PREFIX=x86_64-w64-mingw32- OS_NAME=Windows OS_ARCH=x86_64
mac32:
mac32:
$(MAKE) native OS_NAME=Mac OS_ARCH=x86
linux32:
@ -133,17 +181,16 @@ install-m2:
googlecode-upload: googlecode-lib-upload googlecode-src-upload
googlecode-lib-upload: $(TARGET)/snappy-java-$(VERSION)-lib.upload
googlecode-src-upload: $(TARGET)/snappy-java-$(VERSION)-src.upload
googlecode-lib-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload
googlecode-src-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload
GOOGLECODE_USER:=leo@xerial.org
$(TARGET)/snappy-java-$(VERSION)-lib.upload:
./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).jar
$(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload:
./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).jar
touch $@
$(TARGET)/snappy-java-$(VERSION)-src.upload:
./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).tar.gz
$(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload:
./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).tar.gz
touch $@

59
Makefile.common
View File

@ -8,13 +8,13 @@ endif
JAVA := "$$JAVA_HOME/bin/java"
JAVAC := "$$JAVA_HOME/bin/javac"
JAVAC := "$$JAVA_HOME/bin/javac"
JAVAH := "$$JAVA_HOME/bin/javah"
OSINFO_CLASS := org.xerial.snappy.OSInfo
OSINFO_PROG := lib/org/xerial/snappy/OSInfo.class
## building OSInfo.java
## building OSInfo.java
#$(info compiling OSInfo.java)
#$(shell mkdir -p lib)
#$(shell $(JAVAC) src/main/java/org/xerial/snappy/OSInfo.java -d lib)
@ -31,7 +31,7 @@ else
sep := :
endif
snappy := snappy-$(VERSION)
snappy := snappy-$(SNAPPY_VERSION)
jni_md := $(shell find -L "$(JAVA_HOME)" -name jni_md.h | head -1)
@ -70,7 +70,7 @@ Default_STRIP := $(CROSS_PREFIX)strip
Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
Default_LINKFLAGS := -shared -static
Default_LIBNAME := libsnappyjava.so
Default_SNAPPY_FLAGS :=
Default_SNAPPY_FLAGS :=
Linux-x86_CXX := $(CROSS_PREFIX)g++
Linux-x86_STRIP := $(CROSS_PREFIX)strip
@ -81,18 +81,18 @@ else
endif
Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_LIBNAME := libsnappyjava.so
Linux-x86_SNAPPY_FLAGS:=
Linux-x86_SNAPPY_FLAGS:=
Linux-x86_64_CXX := $(CROSS_PREFIX)g++
Linux-x86_64_CXX := $(CROSS_PREFIX)g++
Linux-x86_64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
endif
Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_64_LIBNAME := libsnappyjava.so
Linux-x86_64_SNAPPY_FLAGS :=
Linux-x86_64_SNAPPY_FLAGS :=
Linux-ppc_CXX := g++
Linux-ppc_STRIP := strip
@ -108,7 +108,7 @@ Linux-ppc_SNAPPY_FLAGS :=
Linux-ppc64_CXX := g++
Linux-ppc64_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
endif
@ -160,19 +160,19 @@ Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390x_LIBNAME := libsnappyjava.so
Linux-s390x_SNAPPY_FLAGS :=
SunOS-x86_CXX := g++
SunOS-x86_CXX := g++
SunOS-x86_STRIP := strip
SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
SunOS-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
SunOS-x86_LIBNAME := libsnappyjava.so
SunOS-x86_SNAPPY_FLAGS :=
SunOS-x86_SNAPPY_FLAGS :=
SunOS-sparc_CXX := g++
SunOS-sparc_CXX := g++
SunOS-sparc_STRIP := strip
SunOS-sparc_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
SunOS-sparc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
SunOS-sparc_LIBNAME := libsnappyjava.so
SunOS-sparc_SNAPPY_FLAGS :=
SunOS-sparc_SNAPPY_FLAGS :=
SunOS-x86_64_CXX := g++
SunOS-x86_64_STRIP := strip
@ -191,14 +191,14 @@ Linux-arm_STRIP := $(CROSS_PREFIX)strip
Linux-arm_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp
Linux-arm_LINKFLAGS := -shared -static-libgcc
Linux-arm_LIBNAME := libsnappyjava.so
Linux-arm_SNAPPY_FLAGS:=
Linux-arm_SNAPPY_FLAGS:=
Linux-armhf_CXX := $(CROSS_PREFIX)g++
Linux-armhf_STRIP := $(CROSS_PREFIX)strip
Linux-armhf_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard
Linux-armhf_LINKFLAGS := -shared -static-libgcc
Linux-armhf_LIBNAME := libsnappyjava.so
Linux-armhf_SNAPPY_FLAGS:=
Linux-armhf_SNAPPY_FLAGS:=
Linux-aarch64_CXX := $(CROSS_PREFIX)g++
Linux-aarch64_STRIP := $(CROSS_PREFIX)strip
@ -214,47 +214,46 @@ Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc
Mac-x86_LIBNAME := libsnappyjava.jnilib
Mac-x86_SNAPPY_FLAGS :=
Mac-x86_64_CXX := g++ -arch $(OS_ARCH)
Mac-x86_64_CXX := g++ -arch $(OS_ARCH)
Mac-x86_64_STRIP := strip -x
Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden
Mac-x86_64_LINKFLAGS := -dynamiclib
Mac-x86_64_LIBNAME := libsnappyjava.jnilib
Mac-x86_64_SNAPPY_FLAGS :=
Mac-x86_64_LINKFLAGS := -dynamiclib
Mac-x86_64_LIBNAME := libsnappyjava.jnilib
Mac-x86_64_SNAPPY_FLAGS :=
FreeBSD-x86_64_CXX := $(CROSS_PREFIX)g++
FreeBSD-x86_64_STRIP := $(CROSS_PREFIX)strip
FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
FreeBSD-x86_64_LINKFLAGS := -shared -static-libgcc
FreeBSD-x86_64_LIBNAME := libsnappyjava.so
FreeBSD-x86_64_SNAPPY_FLAGS :=
FreeBSD-x86_64_SNAPPY_FLAGS :=
Windows-x86_CXX := $(CROSS_PREFIX)g++
Windows-x86_STRIP := $(CROSS_PREFIX)strip
Windows-x86_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_LIBNAME := snappyjava.dll
Windows-x86_SNAPPY_FLAGS :=
Windows-x86_SNAPPY_FLAGS :=
Windows-x86_64_CXX := $(CROSS_PREFIX)g++
Windows-x86_64_STRIP := $(CROSS_PREFIX)strip
Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_64_LIBNAME := snappyjava.dll
Windows-x86_64_SNAPPY_FLAGS :=
Windows-x86_64_SNAPPY_FLAGS :=
CXX := $($(os_arch)_CXX)
STRIP := $($(os_arch)_STRIP)
CXXFLAGS := $($(os_arch)_CXXFLAGS)
LINKFLAGS := $($(os_arch)_LINKFLAGS)
CXXFLAGS := $($(os_arch)_CXXFLAGS)
LINKFLAGS := $($(os_arch)_LINKFLAGS)
LIBNAME := $($(os_arch)_LIBNAME)
SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS)
CXXFLAGS := $(CXXFLAGS) -Ilib/include
CXXFLAGS := $(CXXFLAGS) -Ilib/include
ifneq ($(jni_include),)
CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
endif

6
README.md
View File

@ -143,6 +143,12 @@ Older snapshots of snappy contain a buggy config.h.in that does not work properl
$ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x
$ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1
## Activating SSE2/AVX2 instructions in BitShuffle
The most of the native libraries that snappy-java contains disable SSE2/AVX2 instructions in terms of portability (SSE2 is enabled only in Linux/x86_64 platforms). To enable AVX2 instructions, you need to compile as follows:
$ make CXXFLAGS_BITSHUFFLE=-mavx2 # -msse2 for SSE2 instructions
## Cross-compiling for other platforms
The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM:

184
src/main/java/org/xerial/snappy/BitShuffle.java Normal file
View File

@ -0,0 +1,184 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// BitShuffle.java
// Since: 2016/03/31
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
public class BitShuffle
{
static {
try {
impl = SnappyLoader.loadBitShuffleApi();
}
catch (Exception e) {
throw new ExceptionInInitializerError(e);
}
}
/**
* An instance of BitShuffleNative
*/
private static BitShuffleNative impl;
/**
* Apply a bit-shuffling filter into the input short array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] bitShuffle(short[] input) throws IOException {
byte[] output = new byte[input.length * 2];
int numProcessed = impl.bitShuffle(input, 0, 2, input.length * 2, output, 0);
assert(numProcessed == input.length * 2);
return output;
}
/**
* Apply a bit-shuffling filter into the input int array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] bitShuffle(int[] input) throws IOException {
byte[] output = new byte[input.length * 4];
int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
assert(numProcessed == input.length * 4);
return output;
}
/**
* Apply a bit-shuffling filter into the input long array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] bitShuffle(long[] input) throws IOException {
byte[] output = new byte[input.length * 8];
int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
assert(numProcessed == input.length * 8);
return output;
}
/**
* Apply a bit-shuffling filter into the input float array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] bitShuffle(float[] input) throws IOException {
byte[] output = new byte[input.length * 4];
int numProcessed = impl.bitShuffle(input, 0, 4, input.length * 4, output, 0);
assert(numProcessed == input.length * 4);
return output;
}
/**
* Apply a bit-shuffling filter into the input double array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] bitShuffle(double[] input) throws IOException {
byte[] output = new byte[input.length * 8];
int numProcessed = impl.bitShuffle(input, 0, 8, input.length * 8, output, 0);
assert(numProcessed == input.length * 8);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original short array.
*
* @param input
* @return a short array
* @throws IOException
*/
public static short[] bitUnShuffleShortArray(byte[] input) throws IOException {
short[] output = new short[input.length / 2];
int numProcessed = impl.bitUnShuffle(input, 0, 2, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original int array.
*
* @param input
* @return an int array
* @throws IOException
*/
public static int[] bitUnShuffleIntArray(byte[] input) throws IOException {
int[] output = new int[input.length / 4];
int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original long array.
*
* @param input
* @return a long array
* @throws IOException
*/
public static long[] bitUnShuffleLongArray(byte[] input) throws IOException {
long[] output = new long[input.length / 8];
int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original float array.
*
* @param input
* @return an float array
* @throws IOException
*/
public static float[] bitUnShuffleFloatArray(byte[] input) throws IOException {
float[] output = new float[input.length / 4];
int numProcessed = impl.bitUnShuffle(input, 0, 4, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original double array.
*
* @param input
* @return a double array
* @throws IOException
*/
public static double[] bitUnShuffleDoubleArray(byte[] input) throws IOException {
double[] output = new double[input.length / 8];
int numProcessed = impl.bitUnShuffle(input, 0, 8, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
}

91
src/main/java/org/xerial/snappy/BitShuffleNative.cpp Executable file
View File

@ -0,0 +1,91 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
#include <bitshuffle.h>
#include "BitShuffleNative.h"
inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
{
jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
if(c==0)
return;
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
}
/*
* Class: org_xerial_snappy_SnappyNative
* Method: bitShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
int64_t processedBytes = bshuf_bitshuffle(
in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_SnappyNative
* Method: bitUnShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
int64_t processedBytes = bshuf_bitunshuffle(
in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) processedBytes;
}

29
src/main/java/org/xerial/snappy/BitShuffleNative.h Normal file
View File

@ -0,0 +1,29 @@
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class org_xerial_snappy_BitShuffleNative */
#ifndef _Included_org_xerial_snappy_BitShuffleNative
#define _Included_org_xerial_snappy_BitShuffleNative
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: bitUnShuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_bitUnShuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
#ifdef __cplusplus
}
#endif
#endif

53
src/main/java/org/xerial/snappy/BitShuffleNative.java Normal file
View File

@ -0,0 +1,53 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// BitShuffleNative.java
// Since: 2016/03/31
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
/**
* JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
* defined in BitShuffleNative.h (genereted by javah) and BitShuffleNative.cpp
* <p/>
* <p>
* <b> DO NOT USE THIS CLASS since the direct use of this class might break the
* native library code loading in {@link SnappyLoader}. </b>
* </p>
*
* @author leo
*/
public class BitShuffleNative
{
// ------------------------------------------------------------------------
// Bit-shuffling routines to improve compression of typed binary data.
// A quick benchmark result can be found in a gist below;
// https://gist.github.com/maropu/01103215df34b317a7a7
// ------------------------------------------------------------------------
public native int bitShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
public native int bitUnShuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
}

6
src/main/java/org/xerial/snappy/Snappy.java
View File

@ -44,7 +44,7 @@ public class Snappy
{
static {
try {
impl = SnappyLoader.load();
impl = SnappyLoader.loadSnappyApi();
}
catch (Exception e) {
throw new ExceptionInInitializerError(e);
@ -66,7 +66,7 @@ public class Snappy
public static void cleanUp()
{
SnappyLoader.cleanUpExtractedNativeLib();
SnappyLoader.setApi(null);
SnappyLoader.setSnappyApi(null);
}
/**
@ -295,7 +295,7 @@ public class Snappy
versionData.load(in);
version = versionData.getProperty("version", version);
if (version.equals("unknown")) {
version = versionData.getProperty("VERSION", version);
version = versionData.getProperty("SNAPPY_VERSION", version);
}
version = version.trim().replaceAll("[^0-9\\.]", "");
}

4
src/main/java/org/xerial/snappy/SnappyBundleActivator.java
View File

@ -56,13 +56,13 @@ public class SnappyBundleActivator
library = library.replace(".dylib", ".jnilib");
}
System.loadLibrary(library);
SnappyLoader.setApi(new SnappyNative());
SnappyLoader.setSnappyApi(new SnappyNative());
}
public void stop(BundleContext context)
throws Exception
{
SnappyLoader.setApi(null);
SnappyLoader.setSnappyApi(null);
SnappyLoader.cleanUpExtractedNativeLib();
}
}

71
src/main/java/org/xerial/snappy/SnappyLoader.java
View File

@ -79,8 +79,10 @@ public class SnappyLoader
public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib";
public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility
private static volatile boolean isLoaded = false;
private static volatile SnappyNative api = null;
private static boolean isLoaded = false;
private static volatile SnappyNative snappyApi = null;
private static volatile BitShuffleNative bitshuffleApi = null;
private static File nativeLibFile = null;
@ -91,17 +93,19 @@ public class SnappyLoader
if (!deleted) {
// Deleting native lib has failed, but it's not serious so simply ignore it here
}
snappyApi = null;
bitshuffleApi = null;
}
}
/**
* Set the api instance.
* Set the `snappyApi` instance.
*
* @param nativeCode
*/
static synchronized void setApi(SnappyNative nativeCode)
static synchronized void setSnappyApi(SnappyNative nativeCode)
{
api = nativeCode;
snappyApi = nativeCode;
}
/**
@ -142,40 +146,47 @@ public class SnappyLoader
loadSnappySystemProperties();
}
static synchronized SnappyNative load()
static synchronized SnappyNative loadSnappyApi()
{
if (api != null) {
return api;
if (snappyApi != null) {
return snappyApi;
}
loadNativeLibrary();
setSnappyApi(new SnappyNative());
return snappyApi;
}
try {
loadNativeLibrary();
setApi(new SnappyNative());
isLoaded = true;
static synchronized BitShuffleNative loadBitShuffleApi()
{
if (bitshuffleApi != null) {
return bitshuffleApi;
}
catch (Exception e) {
e.printStackTrace();
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
}
return api;
loadNativeLibrary();
bitshuffleApi = new BitShuffleNative();
return bitshuffleApi;
}
/**
* Load a native library of snappy-java
*/
private static void loadNativeLibrary()
private synchronized static void loadNativeLibrary()
{
nativeLibFile = findNativeLibrary();
if (nativeLibFile != null) {
// Load extracted or specified snappyjava native library.
System.load(nativeLibFile.getAbsolutePath());
}
else {
// Load preinstalled snappyjava (in the path -Djava.library.path)
System.loadLibrary("snappyjava");
if (!isLoaded) {
try {
nativeLibFile = findNativeLibrary();
if (nativeLibFile != null) {
// Load extracted or specified snappyjava native library.
System.load(nativeLibFile.getAbsolutePath());
} else {
// Load preinstalled snappyjava (in the path -Djava.library.path)
System.loadLibrary("snappyjava");
}
}
catch (Exception e) {
e.printStackTrace();
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
}
isLoaded = true;
}
}
@ -372,7 +383,7 @@ public class SnappyLoader
versionData.load(versionFile.openStream());
version = versionData.getProperty("version", version);
if (version.equals("unknown")) {
version = versionData.getProperty("VERSION", version);
version = versionData.getProperty("SNAPPY_VERSION", version);
}
version = version.trim().replaceAll("[^0-9M\\.]", "");
}

30
src/main/java/org/xerial/snappy/SnappyNative.cpp
View File

@ -18,22 +18,22 @@
#include <snappy.h>
#include "SnappyNative.h"
void throw_exception(JNIEnv *env, jobject self, int errorCode)
inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
{
jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
if(c==0)
return;
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
}
JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersion
(JNIEnv * env, jobject self)
{
return env->NewStringUTF("1.1.0");
// TODO: Do we need to read this library version from resources/org/xerial/snappy/VERSION?
return env->NewStringUTF("1.1.3");
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
@ -41,9 +41,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
size_t compressedLength;
snappy::RawCompress((char*) srcAddr, (size_t) length, (char*) destAddr, &compressedLength);
return (jlong) compressedLength;
}
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
(JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
@ -58,9 +56,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
}
return (jlong) uncompressedLength;
}
}
/*
* Class: org_xerial_snappy_Snappy
@ -82,7 +78,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_ni
return (jint) compressedLength;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_lang_Object_2IILjava_lang_Object_2I
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLen, jobject output, jint outputOffset)
{
@ -141,7 +136,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_
return (jint) uncompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: uncompress
@ -168,8 +162,6 @@ JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_
return (jint) decompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: maxCompressedLength
@ -240,8 +232,7 @@ JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__
}
return (jint) result;
}
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
@ -306,4 +297,3 @@ JNIEXPORT void JNICALL Java_org_xerial_snappy_SnappyNative_arrayCopy
env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
}

0
src/main/java/org/xerial/snappy/SnappyNative.h Executable file → Normal file
View File

3
src/main/resources/org/xerial/snappy/VERSION
View File

@ -1 +1,2 @@
VERSION=1.1.3
SNAPPY_VERSION=1.1.3
BITSHUFFLE_VERSION=0.2.2

82
src/test/java/org/xerial/snappy/BitShuffleTest.java Normal file
View File

@ -0,0 +1,82 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// BitShuffleTest.java
// Since: 2016/03/31
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import static org.junit.Assert.*;
import org.junit.Test;
public class BitShuffleTest {
@Test
public void bitShuffleLongArray()
throws Exception
{
long[] data = new long[] {2, 3, 15, 4234, 43251531412342342L, 23423422342L};
byte[] shuffledData = BitShuffle.bitShuffle(data);
long[] result = BitShuffle.bitUnShuffleLongArray(shuffledData);
assertArrayEquals(data, result);
}
@Test
public void bitShuffleShortArray()
throws Exception
{
short[] data = new short[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1};
byte[] shuffledData = BitShuffle.bitShuffle(data);
short[] result = BitShuffle.bitUnShuffleShortArray(shuffledData);
assertArrayEquals(data, result);
}
@Test
public void bitShuffleIntArray()
throws Exception
{
int[] data = new int[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43};
byte[] shuffledData = BitShuffle.bitShuffle(data);
int[] result = BitShuffle.bitUnShuffleIntArray(shuffledData);
assertArrayEquals(data, result);
}
@Test
public void bitShuffleFloatArray()
throws Exception
{
float[] data = new float[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
byte[] shuffledData = BitShuffle.bitShuffle(data);
float[] result = BitShuffle.bitUnShuffleFloatArray(shuffledData);
assertArrayEquals(data, result, 0.0000001f);
}
@Test
public void bitShuffleDoubleArray()
throws Exception
{
double[] data = new double[] {100.0f, 0.5f, -0.1f, 30.3f, Float.MIN_NORMAL, Float.MAX_EXPONENT, Float.MAX_VALUE, -0.1f, Integer.MIN_VALUE};
byte[] shuffledData = BitShuffle.bitShuffle(data);
double[] result = BitShuffle.bitUnShuffleDoubleArray(shuffledData);
assertArrayEquals(data, result, 0.0000001f);
}
}

2
src/test/java/org/xerial/snappy/SnappyLoaderTest.java
View File

@ -117,7 +117,7 @@ public class SnappyLoaderTest
public void load()
throws Exception
{
SnappyLoader.load();
SnappyLoader.loadSnappyApi();
_logger.debug(Snappy.maxCompressedLength(1024));
}