tyler.durden/snappy-java
1
0
Fork 0
mirror of https://github.com/xerial/snappy-java.git synced 2025-04-08 19:35:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Taro L. Saito 2015-05-13 11:06:11 +09:00
commit e5d86ee61e
31 changed files with 1204 additions and 613 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -56,7 +56,7 @@ $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/s
$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $< $(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class $(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class
$(JAVAH) -force -classpath $(TARGET)/classes -o $@ org.xerial.snappy.SnappyNative $(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative
ifndef USE_GIT ifndef USE_GIT
$(SNAPPY_SRC): $(SNAPPY_UNPACKED) $(SNAPPY_SRC): $(SNAPPY_UNPACKED)
@ -135,6 +135,10 @@ linux-arm:
linux-armhf: linux-armhf:
$(MAKE) native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armhf $(MAKE) native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armhf
# for cross-compilation on Ubuntu, install the g++-aarch64-linux-gnu
linux-aarch64:
$(MAKE) native CROSS_PREFIX=aarch64-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64
clean-native-linux32: clean-native-linux32:
$(MAKE) clean-native OS_NAME=Linux OS_ARCH=x86 $(MAKE) clean-native OS_NAME=Linux OS_ARCH=x86

33
Makefile.common
View File

@ -1,6 +1,6 @@
TARGET:=target TARGET:=target
SRC:=src/main/java SRC:=src/main/java
include $(SRC)/org/xerial/snappy/VERSION include src/main/resources/org/xerial/snappy/VERSION
ifndef JAVA_HOME ifndef JAVA_HOME
$(error Set JAVA_HOME environment variable) $(error Set JAVA_HOME environment variable)
@ -42,7 +42,7 @@ endif
# os=Default is meant to be generic unix/linux # os=Default is meant to be generic unix/linux
known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armhf Linux-ppc64 Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armhf Linux-ppc Linux-ppc64 Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc64
os_arch := $(OS_NAME)-$(OS_ARCH) os_arch := $(OS_NAME)-$(OS_ARCH)
IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7")) IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7"))
@ -50,12 +50,14 @@ ifeq (,$(findstring $(strip $(os_arch)),$(known_os_archs)))
os_arch := Default os_arch := Default
endif endif
os_folder := $(shell echo $(OS_NAME) | tr A-Z a-z)
ifneq ($(IBM_JDK_7),) ifneq ($(IBM_JDK_7),)
$(shell mkdir -p $(IBM_JDK_LIB)) $(shell mkdir -p $(IBM_JDK_LIB))
$(shell cp $(JAVA_HOME)/include/jniport.h $(IBM_JDK_LIB)) $(shell cp $(JAVA_HOME)/include/jniport.h $(IBM_JDK_LIB))
$(shell sed -i "s|#define JNIEXPORT *$$|#define JNIEXPORT __attribute__((__visibility__(\"default\")))|" $(IBM_JDK_LIB)/jniport.h) $(shell sed -i "s|#define JNIEXPORT *$$|#define JNIEXPORT __attribute__((__visibility__(\"default\")))|" $(IBM_JDK_LIB)/jniport.h)
$(shell sed -i "s|typedef long long jlong;.*|/*typedef long long jlong;*/|" $(IBM_JDK_LIB)/jniport.h) $(shell sed -i "s|typedef long long jlong;.*|/*typedef long long jlong;*/|" $(IBM_JDK_LIB)/jniport.h)
$(shell cp $(JAVA_HOME)/include/linux/jni_md.h $(IBM_JDK_LIB)) $(shell cp $(JAVA_HOME)/include/$(os_folder)/jni_md.h $(IBM_JDK_LIB))
$(shell sed -i "s|#define JNIEXPORT|#define JNIEXPORT __attribute__((__visibility__(\"default\")))|" $(IBM_JDK_LIB)/jni_md.h) $(shell sed -i "s|#define JNIEXPORT|#define JNIEXPORT __attribute__((__visibility__(\"default\")))|" $(IBM_JDK_LIB)/jni_md.h)
$(shell sed -i "s|typedef long long jlong;.*|/*typedef long long jlong;*/|" $(IBM_JDK_LIB)/jni_md.h) $(shell sed -i "s|typedef long long jlong;.*|/*typedef long long jlong;*/|" $(IBM_JDK_LIB)/jni_md.h)
endif endif
@ -92,17 +94,39 @@ Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_64_LIBNAME := libsnappyjava.so Linux-x86_64_LIBNAME := libsnappyjava.so
Linux-x86_64_SNAPPY_FLAGS := Linux-x86_64_SNAPPY_FLAGS :=
Linux-ppc_CXX := g++
Linux-ppc_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-ppc_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32
else
Linux-ppc_CXXFLAGS := -DHAVE_CONFIG_H -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32
endif
Linux-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc_LIBNAME := libsnappyjava.so
Linux-ppc_SNAPPY_FLAGS :=
Linux-ppc64_CXX := g++ Linux-ppc64_CXX := g++
Linux-ppc64_STRIP := strip Linux-ppc64_STRIP := strip
ifeq ($(IBM_JDK_7),) ifeq ($(IBM_JDK_7),)
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else else
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
endif endif
Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64_LIBNAME := libsnappyjava.so Linux-ppc64_LIBNAME := libsnappyjava.so
Linux-ppc64_SNAPPY_FLAGS := Linux-ppc64_SNAPPY_FLAGS :=
AIX-ppc64_CXX := g++
AIX-ppc64_STRIP := strip -X64
AIX-ppc64_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64
endif
AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc64_SNAPPY_FLAGS :=
SunOS-x86_CXX := g++ SunOS-x86_CXX := g++
SunOS-x86_STRIP := strip SunOS-x86_STRIP := strip
SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
@ -193,3 +217,4 @@ ifneq ($(jni_include),)
CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)" CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
endif endif

22
Milestone.md
View File

@ -1,9 +1,23 @@
## Features under consideration
* `SnappyIndexer` for parallel compression/decompression
* CUI commands (snap/unsnap)
Since vesion 1.1.0.x, Java 6 (1.6) or higher is required. Since vesion 1.1.0.x, Java 6 (1.6) or higher is required.
## snappy-java-1.1.2-RC1 (13 May 2015)
* SnappyInputStream now supports reading concatenated compressed results of SnappyOutputStream
* There has been no compressed format change since 1.0.5.x. So You can read the compressed results interchangeablly between these versions.
* Fixes a problem when java.io.tmpdir does not exist.
## snappy-java-1.1.1.7 (14 Apr 2015)
* Fixes #100
## snappy-java-1.1.1.6 (26 Oct 2014)
* Fixes #88, #89, #90 and #91
* Fixed the broken build of 1.1.1.4 and memory leak bug 1.1.1.5 (so never use these versions)
## snappy-java-1.0.5.4 (12 September 2014)
* Embedded libstdc++ for Linux/amd64 native library (hotfix to 1.0.5.x series)
## snappy-java-1.1.1.3 (19 July 2014)
* Improved the performance of SnappyOutputStream
## snappy-java-1.1.1 (4 July 2014) ## snappy-java-1.1.1 (4 July 2014)
* Added Snappy framing format support: SnappyFramedInput/OutputStream * Added Snappy framing format support: SnappyFramedInput/OutputStream
* Added native libraries for PowerPC, IBM-AIX 6.4, SunOS. * Added native libraries for PowerPC, IBM-AIX 6.4, SunOS.

64
README.md
View File

@ -2,9 +2,11 @@ The snappy-java is a Java port of the snappy
<http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google. <http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google.
## Features ## Features
* Fast compression/decompression tailored to 64-bit CPU architecture. * Fast compression/decompression around 200~400MB/sec.
* Less memory usage. SnappyOutputStream uses only 32KB+ in default.
* JNI-based implementation to achieve comparable performance to the native C++ version. * JNI-based implementation to achieve comparable performance to the native C++ version.
* Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.). * Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.).
* Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
* Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`). * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`. * Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
* [Framing-format support](http://snappy.googlecode.com/svn/trunk/framing_format.txt) (Since 1.1.0 version) * [Framing-format support](http://snappy.googlecode.com/svn/trunk/framing_format.txt) (Since 1.1.0 version)
@ -17,10 +19,8 @@ The snappy-java is a Java port of the snappy
* Here are some [benchmark results](https://github.com/ning/jvm-compressor-benchmark/wiki), comparing * Here are some [benchmark results](https://github.com/ning/jvm-compressor-benchmark/wiki), comparing
snappy-java and the other compressors snappy-java and the other compressors
`LZO-java`/`LZF`/`QuickLZ`/`Gzip`/`Bzip2`. Thanks [Tatu Saloranta @cotowncoder](http://twitter.com/#!/cowtowncoder) for providing the benchmark suite. `LZO-java`/`LZF`/`QuickLZ`/`Gzip`/`Bzip2`. Thanks [Tatu Saloranta @cotowncoder](http://twitter.com/#!/cowtowncoder) for providing the benchmark suite.
* The benchmark result indicates snappy-java is the fastest compreesor/decompressor in Java * The benchmark result indicates snappy-java is the fastest compreesor/decompressor in Java: http://ning.github.com/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html
* <http://ning.github.com/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html> * The decompression speed is twice as fast as the others: http://ning.github.com/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html
* The decompression speed is twice as fast as the others:
* <http://ning.github.com/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html>
## Download ## Download
@ -29,10 +29,27 @@ The snappy-java is a Java port of the snappy
The current stable version is available from here: The current stable version is available from here:
* Release version: http://central.maven.org/maven2/org/xerial/snappy/snappy-java/ * Release version: http://central.maven.org/maven2/org/xerial/snappy/snappy-java/
* (Old archives are here: http://code.google.com/p/snappy-java/downloads/list)
* Snapshot version (the latest beta version): https://oss.sonatype.org/content/repositories/snapshots/org/xerial/snappy/snappy-java/ * Snapshot version (the latest beta version): https://oss.sonatype.org/content/repositories/snapshots/org/xerial/snappy/snappy-java/
For Maven user, see [pom.xml example](#using-with-maven). ### Using with Maven
* Snappy-java is available from Maven's central repository: <http://central.maven.org/maven2/org/xerial/snappy/snappy-java>
Add the following dependency to your pom.xml:
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.2-RC1</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
### Using with sbt
```
libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.2-RC1"
```
## Usage ## Usage
First, import `org.xerial.snapy.Snappy` in your Java code: First, import `org.xerial.snapy.Snappy` in your Java code:
@ -56,9 +73,18 @@ System.out.println(result);
In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used. In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used.
### Stream-based API ### Stream-based API
Stream-based compressor/decompressor `SnappyFramedOutputStream`/`SnappyFramedInputStream` are also available for reading/writing large data sets. Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` are also available for reading/writing large data sets. `SnappyFramedOutputStream`/`SnappyFramedInputStream` can be used for the [framing format](https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt).
* See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.0/snappy-java-1.1.0-javadoc.jar/!/index.html) * See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.1.6/snappy-java-1.1.1.6-javadoc.jar/!/index.html)
#### Compatibility Notes
* `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. Here is the compatibility matrix of data foramt:
| Write\Read | `Snappy.uncompress` | `SnappyInputStream` | `SnappyFramedInputStream` |
| --------------- |:-------------------:|:------------------:|:-----------------------:|
| `Snappy.compress` | ok | ok | x |
| `SnappyOutputStream` | x | ok | x |
| `SnappyFramedOutputStream` | x | x | ok |
### Setting classpath ### Setting classpath
If you have snappy-java-(VERSION).jar in the current directory, use `-classpath` option as follows: If you have snappy-java-(VERSION).jar in the current directory, use `-classpath` option as follows:
@ -68,24 +94,6 @@ If you have snappy-java-(VERSION).jar in the current directory, use `-classpath`
$ javac -classpath ".:snappy-java-(VERSION).jar" Sample.java # in Mac or Linux $ javac -classpath ".:snappy-java-(VERSION).jar" Sample.java # in Mac or Linux
### Using with Maven
* Snappy-java is available from Maven's central repository: <http://repo1.maven.org/maven2/org/xerial/snappy/snappy-java>
Add the following dependency to your pom.xml:
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>(version)</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
### Using with sbt
```
libraryDependencies += "org.xerial.snappy" % "snappy-java" % "(version)"
```
## Public discussion group ## Public discussion group

45
build.sbt
View File

@ -1,6 +1,3 @@
import SonatypeKeys._
sonatypeSettings
name := "snappy-java" name := "snappy-java"
@ -10,7 +7,7 @@ organizationName := "xerial.org"
description := "snappy-java: A fast compression/decompression library" description := "snappy-java: A fast compression/decompression library"
profileName := "org.xerial" sonatypeProfileName := "org.xerial"
pomExtra := { pomExtra := {
<url>https://github.comm/xerial/snappy-java</url> <url>https://github.comm/xerial/snappy-java</url>
@ -47,11 +44,21 @@ pomExtra := {
</scm> </scm>
} }
scalaVersion := "2.11.6"
javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "1.6", "-target", "1.6") javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "1.6", "-target", "1.6")
javacOptions in doc := {
val opts = Seq("-source", "1.6")
if (scala.util.Properties.isJavaAtLeast("1.8"))
opts ++ Seq("-Xdoclint:none")
else
opts
}
testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v") testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v")
//concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1)) concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1))
autoScalaLibrary := false autoScalaLibrary := false
@ -64,7 +71,9 @@ incOptions := incOptions.value.withNameHashing(true)
libraryDependencies ++= Seq( libraryDependencies ++= Seq(
"junit" % "junit" % "4.8.2" % "test", "junit" % "junit" % "4.8.2" % "test",
"org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test", "org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test",
"org.xerial" % "xerial-core" % "1.0.21" % "test", "org.xerial.java" % "xerial-core" % "2.1" % "test",
"org.xerial" % "xerial-core" % "3.2.3" % "test",
"org.scalatest" % "scalatest_2.11" % "2.2.0" % "test",
"org.osgi" % "org.osgi.core" % "4.3.0" % "provided", "org.osgi" % "org.osgi.core" % "4.3.0" % "provided",
"com.novocode" % "junit-interface" % "0.10" % "test" "com.novocode" % "junit-interface" % "0.10" % "test"
) )
@ -72,7 +81,7 @@ libraryDependencies ++= Seq(
osgiSettings osgiSettings
OsgiKeys.exportPackage := Seq("org.xerial.snappy") OsgiKeys.exportPackage := Seq("org.xerial.snappy", "org.xerial.snappy.buffer")
OsgiKeys.bundleSymbolicName := "org.xerial.snappy.snappy-java" OsgiKeys.bundleSymbolicName := "org.xerial.snappy.snappy-java"
@ -88,9 +97,11 @@ OsgiKeys.additionalHeaders := Map(
"org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64", "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64", "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
"org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86", "org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
"org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64",
"org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm", "org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
"org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64", "org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64",
"org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so;osname=linux;processor=ppc64le", "org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so;osname=linux;processor=ppc64le",
"org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
"org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86", "org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
"org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64", "org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
"org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc" "org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc"
@ -100,3 +111,23 @@ OsgiKeys.additionalHeaders := Map(
"Bundle-ActivationPolicy" -> "lazy", "Bundle-ActivationPolicy" -> "lazy",
"Bundle-Name" -> "snappy-java: A fast compression/decompression library" "Bundle-Name" -> "snappy-java: A fast compression/decompression library"
) )
import ReleaseTransformations._
import sbtrelease._
releaseTagName := { (version in ThisBuild).value }
releaseProcess := Seq[ReleaseStep](
checkSnapshotDependencies,
inquireVersions,
runClean,
runTest,
setReleaseVersion,
commitReleaseVersion,
tagRelease,
ReleaseStep(action = Command.process("publishSigned", _)),
setNextVersion,
commitNextVersion,
ReleaseStep(action = Command.process("sonatypeReleaseAll", _)),
pushChanges
)

53
lib/inc_ibm/jni_md.h Normal file
View File

@ -0,0 +1,53 @@
/*===========================================================================
* Licensed Materials - Property of IBM
* "Restricted Materials of IBM"
*
* IBM SDK, Java(tm) Technology Edition, v7
* (C) Copyright IBM Corp. 2014, 2014. All Rights Reserved
*
* US Government Users Restricted Rights - Use, duplication or disclosure
* restricted by GSA ADP Schedule Contract with IBM Corp.
*===========================================================================
*/
/*
* Copyright (c) 1996, 2000, Oracle and/or its affiliates. All rights reserved.
* ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*/
#ifndef _JAVASOFT_JNI_MD_H_
#define _JAVASOFT_JNI_MD_H_
#define JNIEXPORT __attribute__((__visibility__("default")))
#define JNIIMPORT
#define JNICALL
typedef int jint;
#ifdef _LP64 /* 64-bit Solaris */
typedef long jlong;
#else
/*typedef long long jlong;*/
#endif
typedef signed char jbyte;
#endif /* !_JAVASOFT_JNI_MD_H_ */

56
lib/inc_ibm/jniport.h Normal file
View File

@ -0,0 +1,56 @@
/*******************************************************************************
* Licensed Materials - Property of IBM
* "Restricted Materials of IBM"
*
* (c) Copyright IBM Corp. 1991, 2014 All Rights Reserved
*
* US Government Users Restricted Rights - Use, duplication or disclosure
* restricted by GSA ADP Schedule Contract with IBM Corp.
*******************************************************************************/
#ifndef jniport_h
#define jniport_h
#if defined(WIN32) || defined(_WIN32) || defined(RIM386) || (defined(BREW) && defined(AEE_SIMULATOR))
#define JNIEXPORT __declspec(dllexport)
#define JNICALL __stdcall
typedef signed char jbyte;
typedef int jint;
typedef __int64 jlong;
#else
#define JNIEXPORT __attribute__((__visibility__("default")))
typedef signed char jbyte;
/*typedef long long jlong;*/
#ifdef BREW
#include "AEEFile.h"
#define FILE IFile
#endif
typedef int jint;
#endif /* WIN32 */
#ifndef JNICALL
#define JNICALL
#endif
#ifndef JNIEXPORT
#define JNIEXPORT __attribute__((__visibility__("default")))
#endif
#ifndef JNIIMPORT
#define JNIIMPORT
#endif
#ifdef _JNI_IMPLEMENTATION_
#define _JNI_IMPORT_OR_EXPORT_ JNIEXPORT
#else
#define _JNI_IMPORT_OR_EXPORT_ JNIIMPORT
#endif
#endif /* jniport_h */

2
project/build.properties
View File

@ -1,2 +1,2 @@
sbt.version=0.13.5 sbt.version=0.13.8

6
project/plugins.sbt
View File

@ -1,9 +1,9 @@
addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.7.1") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.2.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.5.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.3") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.3.0") addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.3.0")

10
sbt
View File

@ -4,8 +4,8 @@
# Author: Paul Phillips <paulp@typesafe.com> # Author: Paul Phillips <paulp@typesafe.com>
# todo - make this dynamic # todo - make this dynamic
declare -r sbt_release_version="0.13.1" declare -r sbt_release_version="0.13.8"
declare -r sbt_unreleased_version="0.13.2-SNAPSHOT" # -sbt-dev doesn't work at present declare -r sbt_unreleased_version="0.13.8-SNAPSHOT" # -sbt-dev doesn't work at present
declare -r buildProps="project/build.properties" declare -r buildProps="project/build.properties"
declare sbt_jar sbt_dir sbt_create sbt_launch_dir declare sbt_jar sbt_dir sbt_create sbt_launch_dir
@ -119,12 +119,12 @@ init_default_option_file () {
declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation" declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
declare -r default_jvm_opts="-Dfile.encoding=UTF8 -XX:MaxPermSize=384m -Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts" declare -r default_jvm_opts="-ea -Dfile.encoding=UTF8 -Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts"
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
declare -r latest_28="2.8.2" declare -r latest_28="2.8.2"
declare -r latest_29="2.9.3" declare -r latest_29="2.9.3"
declare -r latest_210="2.10.3" declare -r latest_210="2.10.5"
declare -r latest_211="2.11.0-M5" declare -r latest_211="2.11.6"
declare -r script_path="$(get_script_path "$BASH_SOURCE")" declare -r script_path="$(get_script_path "$BASH_SOURCE")"
declare -r script_name="${script_path##*/}" declare -r script_name="${script_path##*/}"

12
src/main/java/org/xerial/snappy/OSInfo.java
View File

@ -43,6 +43,7 @@ public class OSInfo
public static final String IA64_32 = "ia64_32"; public static final String IA64_32 = "ia64_32";
public static final String IA64 = "ia64"; public static final String IA64 = "ia64";
public static final String PPC = "ppc"; public static final String PPC = "ppc";
public static final String PPC64 = "ppc64";
static { static {
// x86 mappings // x86 mappings
@ -75,6 +76,11 @@ public class OSInfo
archMapping.put("power_rs", PPC); archMapping.put("power_rs", PPC);
// TODO: PowerPC 64bit mappings // TODO: PowerPC 64bit mappings
archMapping.put(PPC64, PPC64);
archMapping.put("power64", PPC64);
archMapping.put("powerpc64", PPC64);
archMapping.put("power_pc64", PPC64);
archMapping.put("power_rs64", PPC64);
} }
@ -140,7 +146,11 @@ public class OSInfo
else if (osName.contains("Linux")) { else if (osName.contains("Linux")) {
return "Linux"; return "Linux";
} }
else { else if (osName.contains("AIX")) {
return "AIX";
}
else {
return osName.replaceAll("\\W", ""); return osName.replaceAll("\\W", "");
} }
} }

46
src/main/java/org/xerial/snappy/SnappyCodec.java
View File

@ -50,7 +50,15 @@ import java.util.Arrays;
public class SnappyCodec public class SnappyCodec
{ {
public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 }; public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 };
public static final int MAGIC_LEN = 8; public static final int MAGIC_LEN = MAGIC_HEADER.length;
public static final int HEADER_SIZE = MAGIC_LEN + 8;
public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0);
public static final int MAGIC_HEADER_TAIL = SnappyOutputStream.readInt(MAGIC_HEADER, 4);
static {
assert(MAGIC_HEADER_HEAD < 0);
}
public static final int DEFAULT_VERSION = 1; public static final int DEFAULT_VERSION = 1;
public static final int MINIMUM_COMPATIBLE_VERSION = 1; public static final int MINIMUM_COMPATIBLE_VERSION = 1;
@ -58,11 +66,25 @@ public class SnappyCodec
public final byte[] magic; public final byte[] magic;
public final int version; public final int version;
public final int compatibleVersion; public final int compatibleVersion;
private final byte[] headerArray;
private SnappyCodec(byte[] magic, int version, int compatibleVersion) { private SnappyCodec(byte[] magic, int version, int compatibleVersion) {
this.magic = magic; this.magic = magic;
this.version = version; this.version = version;
this.compatibleVersion = compatibleVersion; this.compatibleVersion = compatibleVersion;
ByteArrayOutputStream header = new ByteArrayOutputStream(HEADER_SIZE);
DataOutputStream d = new DataOutputStream(header);
try {
d.write(magic, 0, MAGIC_LEN);
d.writeInt(version);
d.writeInt(compatibleVersion);
d.close();
}
catch(IOException e) {
throw new RuntimeException(e);
}
headerArray = header.toByteArray();
} }
@Override @Override
@ -71,17 +93,17 @@ public class SnappyCodec
} }
public static int headerSize() { public static int headerSize() {
return MAGIC_LEN + 4 * 2; return HEADER_SIZE;
} }
public void writeHeader(OutputStream out) throws IOException { public int writeHeader(byte[] dst, int dstOffset) {
ByteArrayOutputStream header = new ByteArrayOutputStream(); System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length);
DataOutputStream d = new DataOutputStream(header); return headerArray.length;
d.write(magic, 0, MAGIC_LEN); }
d.writeInt(version);
d.writeInt(compatibleVersion); public int writeHeader(OutputStream out) throws IOException {
d.close(); out.write(headerArray, 0, headerArray.length);
out.write(header.toByteArray(), 0, header.size()); return headerArray.length;
} }
public boolean isValidMagicHeader() { public boolean isValidMagicHeader() {
@ -97,8 +119,6 @@ public class SnappyCodec
return new SnappyCodec(magic, version, compatibleVersion); return new SnappyCodec(magic, version, compatibleVersion);
} }
public static SnappyCodec currentHeader() { public static SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
return new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
}
} }

6
src/main/java/org/xerial/snappy/SnappyErrorCode.java
View File

@ -38,7 +38,11 @@ public enum SnappyErrorCode {
PARSING_ERROR(2), PARSING_ERROR(2),
NOT_A_DIRECT_BUFFER(3), NOT_A_DIRECT_BUFFER(3),
OUT_OF_MEMORY(4), OUT_OF_MEMORY(4),
FAILED_TO_UNCOMPRESS(5); FAILED_TO_UNCOMPRESS(5),
EMPTY_INPUT(6),
INCOMPATIBLE_VERSION(7),
INVALID_CHUNK_SIZE(8)
;
public final int id; public final int id;

23
src/main/java/org/xerial/snappy/SnappyIOException.java Normal file
View File

@ -0,0 +1,23 @@
package org.xerial.snappy;
import java.io.IOException;
/**
* Enhanced IOException with SnappyErrorCode
*/
public class SnappyIOException extends IOException {
private final SnappyErrorCode errorCode;
public SnappyIOException(SnappyErrorCode errorCode, String message) {
super(message);
this.errorCode = errorCode;
}
@Override
public String getMessage() {
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
public SnappyErrorCode getErrorCode() { return errorCode; }
}

104
src/main/java/org/xerial/snappy/SnappyInputStream.java
View File

@ -45,7 +45,7 @@ public class SnappyInputStream extends InputStream
private int uncompressedCursor = 0; private int uncompressedCursor = 0;
private int uncompressedLimit = 0; private int uncompressedLimit = 0;
private byte[] chunkSizeBuf = new byte[4]; private byte[] header = new byte[SnappyCodec.headerSize()];
/** /**
* Create a filter for reading compressed data as a uncompressed stream * Create a filter for reading compressed data as a uncompressed stream
@ -73,7 +73,6 @@ public class SnappyInputStream extends InputStream
} }
protected void readHeader() throws IOException { protected void readHeader() throws IOException {
byte[] header = new byte[SnappyCodec.headerSize()];
int readBytes = 0; int readBytes = 0;
while (readBytes < header.length) { while (readBytes < header.length) {
int ret = in.read(header, readBytes, header.length - readBytes); int ret = in.read(header, readBytes, header.length - readBytes);
@ -82,31 +81,45 @@ public class SnappyInputStream extends InputStream
readBytes += ret; readBytes += ret;
} }
// Quick test of the header // Quick test of the header
if(readBytes == 0) {
// Snappy produces at least 1-byte result. So the empty input is not a valid input
throw new SnappyIOException(SnappyErrorCode.EMPTY_INPUT, "Cannot decompress empty stream");
}
if (readBytes < header.length || header[0] != SnappyCodec.MAGIC_HEADER[0]) { if (readBytes < header.length || header[0] != SnappyCodec.MAGIC_HEADER[0]) {
// do the default uncompression // do the default uncompression
readFully(header, readBytes); readFully(header, readBytes);
return; return;
} }
SnappyCodec codec = SnappyCodec.readHeader(new ByteArrayInputStream(header)); if(!isValidHeader(header)) {
if (codec.isValidMagicHeader()) {
// The input data is compressed by SnappyOutputStream
if (codec.version < SnappyCodec.MINIMUM_COMPATIBLE_VERSION) {
throw new IOException(String.format(
"compressed with imcompatible codec version %d. At least version %d is required",
codec.version, SnappyCodec.MINIMUM_COMPATIBLE_VERSION));
}
}
else {
// (probably) compressed by Snappy.compress(byte[]) // (probably) compressed by Snappy.compress(byte[])
readFully(header, readBytes); readFully(header, readBytes);
return; return;
} }
} }
private static boolean isValidHeader(byte[] header) throws IOException {
SnappyCodec codec = SnappyCodec.readHeader(new ByteArrayInputStream(header));
if (codec.isValidMagicHeader()) {
// The input data is compressed by SnappyOutputStream
if(codec.version < SnappyCodec.MINIMUM_COMPATIBLE_VERSION) {
throw new SnappyIOException(SnappyErrorCode.INCOMPATIBLE_VERSION, String.format(
"Compressed with an incompatible codec version %d. At least version %d is required",
codec.version, SnappyCodec.MINIMUM_COMPATIBLE_VERSION));
}
return true;
}
else
return false;
}
protected void readFully(byte[] fragment, int fragmentLength) throws IOException { protected void readFully(byte[] fragment, int fragmentLength) throws IOException {
// read the entire input data to the buffer if(fragmentLength == 0) {
finishedReading = true;
return;
}
// read the entire input data to the buffer
compressed = new byte[Math.max(8 * 1024, fragmentLength)]; // 8K compressed = new byte[Math.max(8 * 1024, fragmentLength)]; // 8K
System.arraycopy(fragment, 0, compressed, 0, fragmentLength); System.arraycopy(fragment, 0, compressed, 0, fragmentLength);
int cursor = fragmentLength; int cursor = fragmentLength;
@ -316,6 +329,26 @@ public class SnappyInputStream extends InputStream
return read(d, 0, d.length); return read(d, 0, d.length);
} }
/**
* Read next len bytes
* @param dest
* @param offset
* @param len
* @return read bytes
*/
private int readNext(byte[] dest, int offset, int len) throws IOException {
int readBytes = 0;
while (readBytes < len) {
int ret = in.read(dest, readBytes + offset, len - readBytes);
if (ret == -1) {
finishedReading = true;
return readBytes;
}
readBytes += ret;
}
return readBytes;
}
protected boolean hasNextChunk() throws IOException { protected boolean hasNextChunk() throws IOException {
if (finishedReading) if (finishedReading)
return false; return false;
@ -323,16 +356,24 @@ public class SnappyInputStream extends InputStream
uncompressedCursor = 0; uncompressedCursor = 0;
uncompressedLimit = 0; uncompressedLimit = 0;
int readBytes = 0; int readBytes = readNext(header, 0, 4);
while (readBytes < 4) { if(readBytes < 4)
int ret = in.read(chunkSizeBuf, readBytes, 4 - readBytes); return false;
if (ret == -1) {
finishedReading = true; int chunkSize = SnappyOutputStream.readInt(header, 0);
if(chunkSize == SnappyCodec.MAGIC_HEADER_HEAD) {
// Concatenated data
int remainingHeaderSize = SnappyCodec.headerSize() - 4;
readBytes = readNext(header, 4, remainingHeaderSize);
if(readBytes < remainingHeaderSize)
return false;
if(isValidHeader(header))
return hasNextChunk();
else
return false; return false;
}
readBytes += ret;
} }
int chunkSize = SnappyOutputStream.readInt(chunkSizeBuf, 0);
// extend the compressed data buffer size // extend the compressed data buffer size
if (compressed == null || chunkSize > compressed.length) { if (compressed == null || chunkSize > compressed.length) {
compressed = new byte[chunkSize]; compressed = new byte[chunkSize];
@ -347,20 +388,15 @@ public class SnappyInputStream extends InputStream
if (readBytes < chunkSize) { if (readBytes < chunkSize) {
throw new IOException("failed to read chunk"); throw new IOException("failed to read chunk");
} }
try { int uncompressedLength = Snappy.uncompressedLength(compressed, 0, chunkSize);
int uncompressedLength = Snappy.uncompressedLength(compressed, 0, chunkSize); if (uncompressed == null || uncompressedLength > uncompressed.length) {
if (uncompressed == null || uncompressedLength > uncompressed.length) { uncompressed = new byte[uncompressedLength];
uncompressed = new byte[uncompressedLength];
}
int actualUncompressedLength = Snappy.uncompress(compressed, 0, chunkSize, uncompressed, 0);
if (uncompressedLength != actualUncompressedLength) {
throw new IOException("invalid uncompressed byte size");
}
uncompressedLimit = actualUncompressedLength;
} }
catch (IOException e) { int actualUncompressedLength = Snappy.uncompress(compressed, 0, chunkSize, uncompressed, 0);
throw new IOException("failed to uncompress the chunk: " + e.getMessage()); if (uncompressedLength != actualUncompressedLength) {
throw new SnappyIOException(SnappyErrorCode.INVALID_CHUNK_SIZE, String.format("expected %,d bytes, but decompressed chunk has %,d bytes", uncompressedLength, actualUncompressedLength));
} }
uncompressedLimit = actualUncompressedLength;
return true; return true;
} }

8
src/main/java/org/xerial/snappy/SnappyLoader.java
View File

@ -300,11 +300,13 @@ public class SnappyLoader
} }
// Temporary folder for the native lib. Use the value of org.xerial.snappy.tempdir or java.io.tmpdir // Temporary folder for the native lib. Use the value of org.xerial.snappy.tempdir or java.io.tmpdir
String tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR, File tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR, System.getProperty("java.io.tmpdir")));
System.getProperty("java.io.tmpdir"))).getAbsolutePath(); if (!tempFolder.exists()) {
tempFolder.mkdir();
}
// Extract and load a native library inside the jar file // Extract and load a native library inside the jar file
return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder); return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder.getAbsolutePath());
} }

647
src/main/java/org/xerial/snappy/SnappyOutputStream.java
View File

@ -1,315 +1,332 @@
/*-------------------------------------------------------------------------- /*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito * Copyright 2011 Taro L. Saito
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*--------------------------------------------------------------------------*/ *--------------------------------------------------------------------------*/
//-------------------------------------- //--------------------------------------
// XerialJ // XerialJ
// //
// SnappyOutputStream.java // SnappyOutputStream.java
// Since: 2011/03/31 17:44:10 // Since: 2011/03/31 17:44:10
// //
// $URL$ // $URL$
// $Author$ // $Author$
//-------------------------------------- //--------------------------------------
package org.xerial.snappy; package org.xerial.snappy;
import java.io.IOException; import org.xerial.snappy.buffer.BufferAllocatorFactory;
import java.io.OutputStream; import org.xerial.snappy.buffer.BufferAllocator;
import org.xerial.snappy.buffer.CachedBufferAllocator;
/**
* This class implements a stream filter for writing compressed data using import java.io.IOException;
* Snappy. import java.io.OutputStream;
*
* The input data is blocked into 32kb size (in default), and each block is /**
* compressed and then passed to the given {@link OutputStream}. * This class implements a stream filter for writing compressed data using
* * Snappy.
* The output data format is: * <p>
* <ol> * The input data is blocked into 32kb size (in default), and each block is
* <li>snappy codec header defined in {@link SnappyCodec} (8 bytes) * compressed and then passed to the given {@link OutputStream}.
* <li>compressed block 1 : a pair of (compressed data size [4 byte integer. * </p>
* Big-endian], compressed data...) * The output data format is:
* <li>compressed block 2 * <ol>
* <li>... * <li>snappy codec header defined in {@link SnappyCodec} (8 bytes)
* </ol> * <li>compressed block 1 : a pair of (compressed data size [4 byte integer.
* * Big-endian], compressed data...)
* Note that the compressed data created by {@link SnappyOutputStream} cannot be * <li>compressed block 2
* uncompressed by {@link Snappy#uncompress(byte[])} since the output formats of * <li>...
* {@link Snappy#compress(byte[])} and {@link SnappyOutputStream} are different. * </ol>
* Use {@link SnappyInputStream} for uncompress the data generated by * <p/>
* {@link SnappyOutputStream}. * Note that the compressed data created by {@link SnappyOutputStream} cannot be
* * uncompressed by {@link Snappy#uncompress(byte[])} since the output formats of
* @author leo * {@link Snappy#compress(byte[])} and {@link SnappyOutputStream} are different.
* * Use {@link SnappyInputStream} for uncompress the data generated by
*/ * {@link SnappyOutputStream}.
public class SnappyOutputStream extends OutputStream *
{ * @author leo
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size */
public class SnappyOutputStream extends OutputStream {
protected final OutputStream out; static final int MIN_BLOCK_SIZE = 1 * 1024;
private final int blockSize; static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
private int cursor = 0;
protected byte[] uncompressed; protected final OutputStream out;
protected byte[] compressed; private final int blockSize;
public SnappyOutputStream(OutputStream out) throws IOException { private final BufferAllocator inputBufferAllocator;
this(out, DEFAULT_BLOCK_SIZE); private final BufferAllocator outputBufferAllocator;
}
protected final byte[] inputBuffer;
/** protected final byte[] outputBuffer;
* @param out private int inputCursor = 0;
* @param blockSize private int outputCursor = 0;
* byte size of the internal buffer size
* @throws IOException public SnappyOutputStream(OutputStream out) {
*/ this(out, DEFAULT_BLOCK_SIZE);
public SnappyOutputStream(OutputStream out, int blockSize) throws IOException { }
this.out = out;
this.blockSize = blockSize; /**
uncompressed = new byte[blockSize]; * @param out
compressed = new byte[Snappy.maxCompressedLength(blockSize)]; * @param blockSize byte size of the internal buffer size
writeHeader(); * @throws IOException
} */
public SnappyOutputStream(OutputStream out, int blockSize) {
protected void writeHeader() throws IOException { this(out, blockSize, CachedBufferAllocator.factory);
SnappyCodec.currentHeader().writeHeader(out); }
}
public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactory bufferAllocatorFactory) {
/** this.out = out;
* Writes len bytes from the specified byte array starting at offset off to this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
* this output stream. The general contract for write(b, off, len) is that int inputSize = blockSize;
* some of the bytes in the array b are written to the output stream in int outputSize = SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(blockSize);
* order; element b[off] is the first byte written and b[off+len-1] is the
* last byte written by this operation. this.inputBufferAllocator = bufferAllocatorFactory.getBufferAllocator(inputSize);
*/ this.outputBufferAllocator = bufferAllocatorFactory.getBufferAllocator(outputSize);
/* (non-Javadoc)
* @see java.io.OutputStream#write(byte[], int, int) inputBuffer = inputBufferAllocator.allocate(inputSize);
*/ outputBuffer = outputBufferAllocator.allocate(outputSize);
@Override
public void write(byte[] b, int off, int len) throws IOException { outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
rawWrite(b, off, len); }
}
/** /* (non-Javadoc)
* Compress the input long array data * @see java.io.OutputStream#write(byte[], int, int)
* */
* @param d @Override
* input array public void write(byte[] b, int off, int len) throws IOException {
* @param off rawWrite(b, off, len);
* offset in the array }
* @param len
* the number of elements in the array to copy /**
* @throws IOException * Compress the input long array data
*/ *
public void write(long[] d, int off, int len) throws IOException { * @param d input array
rawWrite(d, off * 8, len * 8); * @param off offset in the array
} * @param len the number of elements in the array to copy
* @throws IOException
/** */
* Compress the input double array data public void write(long[] d, int off, int len) throws IOException {
* rawWrite(d, off * 8, len * 8);
* @param f }
* input array
* @param off /**
* offset in the array * Compress the input double array data
* @param len *
* the number of elements in the array to copy * @param f input array
* @throws IOException * @param off offset in the array
*/ * @param len the number of elements in the array to copy
public void write(double[] f, int off, int len) throws IOException { * @throws IOException
rawWrite(f, off * 8, len * 8); */
} public void write(double[] f, int off, int len) throws IOException {
rawWrite(f, off * 8, len * 8);
/** }
* Compress the input float array data
* /**
* @param f * Compress the input float array data
* input array *
* @param off * @param f input array
* offset in the array * @param off offset in the array
* @param len * @param len the number of elements in the array to copy
* the number of elements in the array to copy * @throws IOException
* @throws IOException */
*/ public void write(float[] f, int off, int len) throws IOException {
public void write(float[] f, int off, int len) throws IOException { rawWrite(f, off * 4, len * 4);
rawWrite(f, off * 4, len * 4); }
}
/**
/** * Compress the input int array data
* Compress the input int array data *
* * @param f input array
* @param f * @param off offset in the array
* input array * @param len the number of elements in the array to copy
* @param off * @throws IOException
* offset in the array */
* @param len public void write(int[] f, int off, int len) throws IOException {
* the number of elements in the array to copy rawWrite(f, off * 4, len * 4);
* @throws IOException }
*/
public void write(int[] f, int off, int len) throws IOException { /**
rawWrite(f, off * 4, len * 4); * Compress the input short array data
} *
* @param f input array
/** * @param off offset in the array
* Compress the input short array data * @param len the number of elements in the array to copy
* * @throws IOException
* @param f */
* input array public void write(short[] f, int off, int len) throws IOException {
* @param off rawWrite(f, off * 2, len * 2);
* offset in the array }
* @param len
* the number of elements in the array to copy /**
* @throws IOException * Compress the input array data
*/ *
public void write(short[] f, int off, int len) throws IOException { * @param d
rawWrite(f, off * 2, len * 2); * @throws IOException
} */
public void write(long[] d) throws IOException {
/** write(d, 0, d.length);
* Compress the input array data }
*
* @param d /**
* @throws IOException * Compress the input array data
*/ *
public void write(long[] d) throws IOException { * @param f
write(d, 0, d.length); * @throws IOException
} */
public void write(double[] f) throws IOException {
/** write(f, 0, f.length);
* Compress the input array data }
*
* @param f /**
* @throws IOException * Compress the input array data
*/ *
public void write(double[] f) throws IOException { * @param f
write(f, 0, f.length); * @throws IOException
} */
public void write(float[] f) throws IOException {
/** write(f, 0, f.length);
* Compress the input array data }
*
* @param f /**
* @throws IOException * Compress the input array data
*/ *
public void write(float[] f) throws IOException { * @param f
write(f, 0, f.length); * @throws IOException
} */
public void write(int[] f) throws IOException {
/** write(f, 0, f.length);
* Compress the input array data }
*
* @param f /**
* @throws IOException * Compress the input array data
*/ *
public void write(int[] f) throws IOException { * @param f
write(f, 0, f.length); * @throws IOException
} */
public void write(short[] f) throws IOException {
/** write(f, 0, f.length);
* Compress the input array data }
*
* @param f private boolean hasSufficientOutputBufferFor(int inputSize) {
* @throws IOException int maxCompressedSize = Snappy.maxCompressedLength(inputSize);
*/ return maxCompressedSize < outputBuffer.length - outputCursor - 4;
public void write(short[] f) throws IOException { }
write(f, 0, f.length);
} /**
* Compress the raw byte array data.
/** *
* Compress the raw byte array data. * @param array array data of any type (e.g., byte[], float[], long[], ...)
* * @param byteOffset
* @param array * @param byteLength
* array data of any type (e.g., byte[], float[], long[], ...) * @throws IOException
* @param byteOffset */
* @param byteLength public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException {
* @throws IOException int cursor = 0;
*/ while(cursor < byteLength) {
public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException { int readLen = Math.min(byteLength - cursor, blockSize - inputCursor);
for (int readBytes = 0; readBytes < byteLength;) { // copy the input data to uncompressed buffer
int copyLen = Math.min(uncompressed.length - cursor, byteLength - readBytes); if(readLen > 0) {
Snappy.arrayCopy(array, byteOffset + readBytes, copyLen, uncompressed, cursor); Snappy.arrayCopy(array, byteOffset + cursor, readLen, inputBuffer, inputCursor);
readBytes += copyLen; inputCursor += readLen;
cursor += copyLen; }
if(inputCursor < blockSize)
if (cursor >= uncompressed.length) { return;
dump();
} compressInput();
} cursor += readLen;
} }
}
/**
* Writes the specified byte to this output stream. The general contract for /**
* write is that one byte is written to the output stream. The byte to be * Writes the specified byte to this output stream. The general contract for
* written is the eight low-order bits of the argument b. The 24 high-order * write is that one byte is written to the output stream. The byte to be
* bits of b are ignored. * written is the eight low-order bits of the argument b. The 24 high-order
*/ * bits of b are ignored.
/* (non-Javadoc) */
* @see java.io.OutputStream#write(int) /* (non-Javadoc)
*/ * @see java.io.OutputStream#write(int)
@Override */
public void write(int b) throws IOException { @Override
if (cursor >= uncompressed.length) { public void write(int b) throws IOException {
dump(); if(inputCursor >= inputBuffer.length) {
} compressInput();
uncompressed[cursor++] = (byte) b; }
} inputBuffer[inputCursor++] = (byte) b;
}
/* (non-Javadoc)
* @see java.io.OutputStream#flush() /* (non-Javadoc)
*/ * @see java.io.OutputStream#flush()
@Override */
public void flush() throws IOException { @Override
dump(); public void flush() throws IOException {
out.flush(); compressInput();
} dumpOutput();
out.flush();
static void writeInt(OutputStream out, int value) throws IOException { }
out.write((value >> 24) & 0xFF);
out.write((value >> 16) & 0xFF); static void writeInt(byte[] dst, int offset, int v) {
out.write((value >> 8) & 0xFF); dst[offset] = (byte) ((v >> 24) & 0xFF);
out.write((value >> 0) & 0xFF); dst[offset + 1] = (byte) ((v >> 16) & 0xFF);
} dst[offset + 2] = (byte) ((v >> 8) & 0xFF);
dst[offset + 3] = (byte) ((v >> 0) & 0xFF);
static int readInt(byte[] buffer, int pos) { }
int b1 = (buffer[pos] & 0xFF) << 24;
int b2 = (buffer[pos + 1] & 0xFF) << 16; static int readInt(byte[] buffer, int pos) {
int b3 = (buffer[pos + 2] & 0xFF) << 8; int b1 = (buffer[pos] & 0xFF) << 24;
int b4 = buffer[pos + 3] & 0xFF; int b2 = (buffer[pos + 1] & 0xFF) << 16;
return b1 | b2 | b3 | b4; int b3 = (buffer[pos + 2] & 0xFF) << 8;
} int b4 = buffer[pos + 3] & 0xFF;
return b1 | b2 | b3 | b4;
protected void dump() throws IOException { }
if (cursor <= 0)
return; // no need to dump protected void dumpOutput() throws IOException {
if(outputCursor > 0) {
// Compress and dump the buffer content out.write(outputBuffer, 0, outputCursor);
int compressedSize = Snappy.compress(uncompressed, 0, cursor, compressed, 0); outputCursor = 0;
writeInt(out, compressedSize); }
out.write(compressed, 0, compressedSize); }
cursor = 0;
} protected void compressInput() throws IOException {
if(inputCursor <= 0) {
/** return; // no need to dump
* close the stream }
*/
/* (non-Javadoc) // Compress and dump the buffer content
* @see java.io.OutputStream#close() if(!hasSufficientOutputBufferFor(inputCursor)) {
*/ dumpOutput();
@Override }
public void close() throws IOException { int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4);
flush(); // Write compressed data size
writeInt(outputBuffer, outputCursor, compressedSize);
super.close(); outputCursor += 4 + compressedSize;
out.close(); inputCursor = 0;
} }
} /**
* close the stream
*/
/* (non-Javadoc)
* @see java.io.OutputStream#close()
*/
@Override
public void close() throws IOException {
try {
flush();
out.close();
} finally {
inputBufferAllocator.release(inputBuffer);
outputBufferAllocator.release(outputBuffer);
}
}
}

2
src/main/java/org/xerial/snappy/VERSION
View File

@ -1,2 +0,0 @@
VERSION=1.1.1

11
src/main/java/org/xerial/snappy/buffer/BufferAllocator.java Normal file
View File

@ -0,0 +1,11 @@
package org.xerial.snappy.buffer;
/**
* BufferAllocator interface. The implementation of this interface must be thread-safe
*/
public interface BufferAllocator {
public byte[] allocate(int size);
public void release(byte[] buffer);
}

10
src/main/java/org/xerial/snappy/buffer/BufferAllocatorFactory.java Normal file
View File

@ -0,0 +1,10 @@
package org.xerial.snappy.buffer;
/**
*
*/
public interface BufferAllocatorFactory {
BufferAllocator getBufferAllocator(int minSize);
}

61
src/main/java/org/xerial/snappy/buffer/CachedBufferAllocator.java Normal file
View File

@ -0,0 +1,61 @@
package org.xerial.snappy.buffer;
import java.lang.ref.SoftReference;
import java.util.*;
/**
* Cached buffer
*/
public class CachedBufferAllocator implements BufferAllocator {
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() {
@Override
public BufferAllocator getBufferAllocator(int bufferSize) {
return CachedBufferAllocator.getAllocator(bufferSize);
}
};
/**
* Use SoftReference so that having this queueTable does not prevent the GC of CachedBufferAllocator instances
*/
public static Map<Integer, SoftReference<CachedBufferAllocator>> queueTable = new HashMap<Integer, SoftReference<CachedBufferAllocator>>();
private final int bufferSize;
private final Deque<byte[]> bufferQueue;
public CachedBufferAllocator(int bufferSize) {
this.bufferSize = bufferSize;
this.bufferQueue = new ArrayDeque<byte[]>();
}
public static synchronized CachedBufferAllocator getAllocator(int bufferSize) {
CachedBufferAllocator result = null;
if (queueTable.containsKey(bufferSize)) {
result = queueTable.get(bufferSize).get();
}
if (result == null) {
result = new CachedBufferAllocator(bufferSize);
queueTable.put(bufferSize, new SoftReference<CachedBufferAllocator>(result));
}
return result;
}
@Override
public byte[] allocate(int size) {
synchronized(this) {
if(bufferQueue.isEmpty()) {
return new byte[size];
}
else {
return bufferQueue.pollFirst();
}
}
}
@Override
public void release(byte[] buffer) {
synchronized(this) {
bufferQueue.addLast(buffer);
}
}
}

26
src/main/java/org/xerial/snappy/buffer/DefaultBufferAllocator.java Normal file
View File

@ -0,0 +1,26 @@
package org.xerial.snappy.buffer;
/**
* Simple buffer allocator, which does not reuse the allocated buffer
*/
public class DefaultBufferAllocator implements BufferAllocator {
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() {
public BufferAllocator singleton = new DefaultBufferAllocator();
@Override
public BufferAllocator getBufferAllocator(int bufferSize) {
return singleton;
}
};
@Override
public byte[] allocate(int size) {
return new byte[size];
}
@Override
public void release(byte[] buffer) {
// do nothing
}
}

1
src/main/resources/org/xerial/snappy/VERSION Executable file
View File

@ -0,0 +1 @@
VERSION=1.1.2

BIN
src/main/resources/org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a Executable file
View File

Binary file not shown.

BIN
src/main/resources/org/xerial/snappy/native/AIX/ppc64/libsnappyjava.so
View File

Binary file not shown.

BIN
src/main/resources/org/xerial/snappy/native/Linux/armhf/libsnappyjava.so
View File

Binary file not shown.

51
src/test/java/org/xerial/snappy/SnappyInputStreamTest.java
View File

@ -35,6 +35,7 @@ import java.io.InputStream;
import org.junit.Test; import org.junit.Test;
import org.xerial.util.FileResource; import org.xerial.util.FileResource;
import org.xerial.util.log.Logger; import org.xerial.util.log.Logger;
import scala.Array;
public class SnappyInputStreamTest public class SnappyInputStreamTest
{ {
@ -62,7 +63,7 @@ public class SnappyInputStreamTest
} }
} }
public static byte[] biteWiseReadFully(InputStream input) throws IOException { public static byte[] byteWiseReadFully(InputStream input) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buf = new byte[4096]; byte[] buf = new byte[4096];
for (int readData = 0; (readData = input.read()) != -1;) { for (int readData = 0; (readData = input.read()) != -1;) {
@ -108,7 +109,7 @@ public class SnappyInputStreamTest
byte[] compressed = Snappy.compress(orig); byte[] compressed = Snappy.compress(orig);
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed));
byte[] uncompressed = biteWiseReadFully(in); byte[] uncompressed = byteWiseReadFully(in);
assertEquals(orig.length, uncompressed.length); assertEquals(orig.length, uncompressed.length);
assertArrayEquals(orig, uncompressed); assertArrayEquals(orig, uncompressed);
@ -129,4 +130,50 @@ public class SnappyInputStreamTest
in.close(); in.close();
} }
@Test
public void emptyStream() throws Exception {
try {
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(new byte[0]));
byte[] uncompressed = readFully(in);
assertEquals(0, uncompressed.length);
fail("should not reach here");
}
catch(SnappyIOException e) {
assertEquals(SnappyErrorCode.EMPTY_INPUT, e.getErrorCode());
}
}
public static byte[] compressResource(String resourcePath) throws Exception {
ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream();
SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf);
byte[] orig = readResourceFile(resourcePath);
snappyOut.write(orig);
snappyOut.close();
return compressedBuf.toByteArray();
}
@Test
public void chunkRead() throws Exception {
byte[] chunk1 = compressResource("alice29.txt");
byte[] chunk2 = compressResource("testdata/calgary/paper6");
byte[] concatenated = new byte[chunk1.length + chunk2.length];
System.arraycopy(chunk1, 0, concatenated, 0, chunk1.length);
System.arraycopy(chunk2, 0, concatenated, chunk1.length, chunk2.length);
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(concatenated));
byte[] uncompressed = readFully(in);
byte[] orig1 = readResourceFile("alice29.txt");
byte[] orig2 = readResourceFile("testdata/calgary/paper6");
assertEquals(orig1.length + orig2.length, uncompressed.length);
byte[] uncompressed1 = new byte[orig1.length];
byte[] uncompressed2 = new byte[orig2.length];
System.arraycopy(uncompressed, 0, uncompressed1, 0, orig1.length);
System.arraycopy(uncompressed, orig1.length, uncompressed2, 0, orig2.length);
assertArrayEquals(orig1, uncompressed1);
assertArrayEquals(orig2, uncompressed2);
}
} }

438
src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java
View File

@ -1,188 +1,250 @@
/*-------------------------------------------------------------------------- /*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito * Copyright 2011 Taro L. Saito
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*--------------------------------------------------------------------------*/ *--------------------------------------------------------------------------*/
//-------------------------------------- //--------------------------------------
// XerialJ // XerialJ
// //
// SnappyOutputStreamTest.java // SnappyOutputStreamTest.java
// Since: 2011/03/31 18:26:31 // Since: 2011/03/31 18:26:31
// //
// $URL$ // $URL$
// $Author$ // $Author$
//-------------------------------------- //--------------------------------------
package org.xerial.snappy; package org.xerial.snappy;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import org.junit.Test; import org.junit.Test;
import org.xerial.util.FileResource; import org.xerial.util.FileResource;
import org.xerial.util.log.Logger; import org.xerial.util.log.Logger;
public class SnappyOutputStreamTest public class SnappyOutputStreamTest
{ {
private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class); private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class);
@Test @Test
public void test() throws Exception { public void test() throws Exception {
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
SnappyOutputStream sout = new SnappyOutputStream(buf); SnappyOutputStream sout = new SnappyOutputStream(buf);
BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class, BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class,
"alice29.txt").openStream()); "alice29.txt").openStream());
assertNotNull(input); assertNotNull(input);
ByteArrayOutputStream orig = new ByteArrayOutputStream(); ByteArrayOutputStream orig = new ByteArrayOutputStream();
byte[] tmp = new byte[1024]; byte[] tmp = new byte[1024];
for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) { for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) {
sout.write(tmp, 0, readBytes); sout.write(tmp, 0, readBytes);
orig.write(tmp, 0, readBytes); // preserve the original data orig.write(tmp, 0, readBytes); // preserve the original data
} }
input.close(); input.close();
sout.flush(); sout.flush();
orig.flush(); orig.flush();
int compressedSize = buf.size(); int compressedSize = buf.size();
_logger.debug("compressed size: " + compressedSize); _logger.debug("compressed size: " + compressedSize);
ByteArrayOutputStream decompressed = new ByteArrayOutputStream(); ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
byte[] compressed = buf.toByteArray(); byte[] compressed = buf.toByteArray();
// decompress // decompress
for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length;) { for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length;) {
int chunkSize = SnappyOutputStream.readInt(compressed, cursor); int chunkSize = SnappyOutputStream.readInt(compressed, cursor);
cursor += 4; cursor += 4;
byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)]; byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)];
int decompressedSize = Snappy.uncompress(compressed, cursor, chunkSize, tmpOut, 0); int decompressedSize = Snappy.uncompress(compressed, cursor, chunkSize, tmpOut, 0);
cursor += chunkSize; cursor += chunkSize;
decompressed.write(tmpOut); decompressed.write(tmpOut);
} }
decompressed.flush(); decompressed.flush();
assertEquals(orig.size(), decompressed.size()); assertEquals(orig.size(), decompressed.size());
assertArrayEquals(orig.toByteArray(), decompressed.toByteArray()); assertArrayEquals(orig.toByteArray(), decompressed.toByteArray());
} }
@Test @Test
public void bufferSize() throws Exception { public void bufferSize() throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b, 500); SnappyOutputStream os = new SnappyOutputStream(b, 1500);
final int bytesToWrite = 5000; final int bytesToWrite = 5000;
byte[] orig = new byte[bytesToWrite]; byte[] orig = new byte[bytesToWrite];
for (int i = 0; i < 5000; ++i) { for (int i = 0; i < 5000; ++i) {
byte v = (byte) (i % 128); byte v = (byte) (i % 128);
orig[i] = v; orig[i] = v;
os.write(v); os.write(v);
} }
os.close(); os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
byte[] buf = new byte[bytesToWrite / 101]; byte[] buf = new byte[bytesToWrite / 101];
while (is.read(buf) != -1) {} while (is.read(buf) != -1) {}
is.close(); is.close();
} }
@Test @Test
public void longArrayCompress() throws Exception { public void smallWrites() throws Exception {
long[] l = new long[10];
for (int i = 0; i < l.length; ++i) { byte[] orig = CalgaryTest.readFile("alice29.txt");
l[i] = i % 3 + i * 11; ByteArrayOutputStream b = new ByteArrayOutputStream();
} SnappyOutputStream out = new SnappyOutputStream(b);
ByteArrayOutputStream b = new ByteArrayOutputStream(); for(byte c : orig) {
SnappyOutputStream os = new SnappyOutputStream(b); out.write(c);
}
os.write(l); out.close();
os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
long[] l2 = new long[10]; byte[] decompressed = new byte[orig.length];
int readBytes = is.read(l2); int cursor = 0;
is.close(); int readLen = 0;
for(int i=0; i < decompressed.length && (readLen = is.read(decompressed, i, decompressed.length-i)) != -1; ) {
assertEquals(10 * 8, readBytes); i += readLen;
assertArrayEquals(l, l2); }
is.close();
} assertArrayEquals(orig, decompressed);
}
@Test
public void writeDoubleArray() throws Exception { /**
ByteArrayOutputStream b = new ByteArrayOutputStream(); * Compress the input array by passing it chunk-by-chunk to a SnappyOutputStream.
SnappyOutputStream os = new SnappyOutputStream(b); * @param orig the data to compress
* @param maxChunkSize the maximum chunk size, in bytes.
double[] orig = new double[] { 1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20 }; * @return the compressed bytes
os.write(orig); */
os.close(); private static byte[] compressAsChunks(byte[] orig, int maxChunkSize) throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); SnappyOutputStream out = new SnappyOutputStream(b);
double[] uncompressed = new double[orig.length];
is.read(uncompressed); int remaining = orig.length;
is.close(); for (int start = 0; start < orig.length; start += maxChunkSize) {
out.write(orig, start, remaining < maxChunkSize ? remaining : maxChunkSize);
assertArrayEquals(orig, uncompressed, 0.0); remaining -= maxChunkSize;
} }
out.close();
@Test return b.toByteArray();
public void writeFloatArray() throws Exception { }
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b); @Test
public void batchingOfWritesShouldNotAffectCompressedDataSize() throws Exception {
float[] orig = new float[] { 1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f }; // Regression test for issue #100, a bug where the size of compressed data could be affected
os.write(orig); // by the batching of writes to the SnappyOutputStream rather than the total amount of data
os.close(); // written to the stream.
byte[] orig = CalgaryTest.readFile("alice29.txt");
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); // Compress the data once so that we know the expected size:
float[] uncompressed = new float[orig.length]; byte[] expectedCompressedData = compressAsChunks(orig, Integer.MAX_VALUE);
is.read(uncompressed); // Hardcoding an expected compressed size here will catch regressions that lower the
is.close(); // compression quality:
assertEquals(91013, expectedCompressedData.length);
assertArrayEquals(orig, uncompressed, 0.0f); // The chunk size should not affect the size of the compressed output:
} int[] chunkSizes = new int[] { 1, 100, 1023, 1024, 10000};
for (int chunkSize : chunkSizes) {
@Test byte[] compressedData = compressAsChunks(orig, chunkSize);
public void writeIntArray() throws Exception { assertEquals(String.format("when chunk size = %,d", chunkSize), expectedCompressedData.length, compressedData.length);
ByteArrayOutputStream b = new ByteArrayOutputStream(); assertArrayEquals(expectedCompressedData, compressedData);
SnappyOutputStream os = new SnappyOutputStream(b); }
}
int[] orig = new int[] { 0, -1, -34, 43, 234, 34324, -234 };
os.write(orig); @Test
os.close(); public void longArrayCompress() throws Exception {
long[] l = new long[10];
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); for (int i = 0; i < l.length; ++i) {
int[] uncompressed = new int[orig.length]; l[i] = i % 3 + i * 11;
is.read(uncompressed); }
is.close();
ByteArrayOutputStream b = new ByteArrayOutputStream();
assertArrayEquals(orig, uncompressed); SnappyOutputStream os = new SnappyOutputStream(b);
}
os.write(l);
@Test os.close();
public void writeShortArray() throws Exception { SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
ByteArrayOutputStream b = new ByteArrayOutputStream(); long[] l2 = new long[10];
SnappyOutputStream os = new SnappyOutputStream(b); int readBytes = is.read(l2);
is.close();
short[] orig = new short[] { 0, -1, -34, 43, 234, 324, -234 };
os.write(orig); assertEquals(10 * 8, readBytes);
os.close(); assertArrayEquals(l, l2);
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); }
short[] uncompressed = new short[orig.length];
is.read(uncompressed); @Test
is.close(); public void writeDoubleArray() throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream();
assertArrayEquals(orig, uncompressed); SnappyOutputStream os = new SnappyOutputStream(b);
}
double[] orig = new double[] { 1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20 };
} os.write(orig);
os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
double[] uncompressed = new double[orig.length];
is.read(uncompressed);
is.close();
assertArrayEquals(orig, uncompressed, 0.0);
}
@Test
public void writeFloatArray() throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b);
float[] orig = new float[] { 1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f };
os.write(orig);
os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
float[] uncompressed = new float[orig.length];
is.read(uncompressed);
is.close();
assertArrayEquals(orig, uncompressed, 0.0f);
}
@Test
public void writeIntArray() throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b);
int[] orig = new int[] { 0, -1, -34, 43, 234, 34324, -234 };
os.write(orig);
os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
int[] uncompressed = new int[orig.length];
is.read(uncompressed);
is.close();
assertArrayEquals(orig, uncompressed);
}
@Test
public void writeShortArray() throws Exception {
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b);
short[] orig = new short[] { 0, -1, -34, 43, 234, 324, -234 };
os.write(orig);
os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
short[] uncompressed = new short[orig.length];
is.read(uncompressed);
is.close();
assertArrayEquals(orig, uncompressed);
}
}

50
src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala Normal file
View File

@ -0,0 +1,50 @@
package org.xerial.snappy
import java.io.{ByteArrayOutputStream, ByteArrayInputStream}
import xerial.core.log.LogLevel
import scala.util.Random
/**
*
*/
class SnappyPerformanceTest extends SnappySpec {
lazy val data = {
val a = new Array[Byte](128 * 1024 * 1024)
for (i <- (0 until a.length).par) {
a(i) = Math.sin(i * 0.01).toByte
}
a
}
"SnappyOutputStream" should {
"improve output performance" taggedAs("out") in {
val input = data
time("compression", repeat=100, logLevel = LogLevel.INFO) {
// 0.037 sec. => 0.026
block("default") {
val out = new ByteArrayOutputStream()
val sout = new SnappyOutputStream(out)
sout.write(input)
sout.close()
out.close()
}
}
//info(f"compressed size: ${compressed.length}%,d, input: ${data.length}%,d")
}
}
}

22
src/test/scala/org/xerial/snappy/SnappySpec.scala Normal file
View File

@ -0,0 +1,22 @@
package org.xerial.snappy
import org.scalatest._
import xerial.core.log.Logger
import xerial.core.util.Timer
/**
*
*/
trait SnappySpec
extends WordSpec
with Matchers
with GivenWhenThen
with OptionValues
with BeforeAndAfter
with Timer
with Logger
{
implicit def toTag(s:String) : Tag = Tag(s)
}

2
version.sbt
View File

@ -1 +1 @@
version in ThisBuild := "1.1.1" version in ThisBuild := "1.1.2-SNAPSHOT"