tyler.durden/snappy-java
1
0
Fork 0
mirror of https://github.com/xerial/snappy-java.git synced 2025-07-21 04:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Taro L. Saito 2016-02-16 08:26:22 -08:00
commit fd07ae7afe
50 changed files with 4937 additions and 4097 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,6 +0,0 @@
#Tue Mar 29 16:49:37 JST 2011
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.6

4
.settings/org.eclipse.jdt.ui.prefs
View File

File diff suppressed because one or more lines are too long

9
.settings/org.maven.ide.eclipse.prefs
View File

@ -1,9 +0,0 @@
#Tue Mar 29 16:49:37 JST 2011
activeProfiles=
eclipse.preferences.version=1
fullBuildGoals=process-test-resources
includeModules=false
resolveWorkspaceProjects=true
resourceFilterGoals=process-resources resources\:testResources
skipCompilerPlugin=true
version=1

5
.travis.yml
View File

@ -1,6 +1,9 @@
language: scala language: scala
sudo: false
scala: scala:
- 2.11.1 - 2.11.7
jdk: jdk:
- openjdk6 - openjdk6
- openjdk7 - openjdk7

4
Makefile
View File

@ -35,7 +35,7 @@ endif
$(SNAPPY_ARCHIVE): $(SNAPPY_ARCHIVE):
@mkdir -p $(@D) @mkdir -p $(@D)
curl -o$@ http://snappy.googlecode.com/files/snappy-$(VERSION).tar.gz curl -L -o$@ https://github.com/google/snappy/releases/download/$(VERSION)/snappy-$(VERSION).tar.gz
$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE) $(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET) $(TAR) xvfz $< -C $(TARGET)
@ -47,7 +47,7 @@ $(SNAPPY_GIT_UNPACKED):
git clone $(GIT_REPO_URL) $(SNAPPY_SRC_DIR) git clone $(GIT_REPO_URL) $(SNAPPY_SRC_DIR)
git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(GIT_SNAPPY_BRANCH) $(GIT_SNAPPY_BRANCH) git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(GIT_SNAPPY_BRANCH) $(GIT_SNAPPY_BRANCH)
touch $@ touch $@
cd $(SNAPPY_SRC_DIR) && ./configure cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure
jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h

35
Makefile.common
View File

@ -42,7 +42,7 @@ endif
# os=Default is meant to be generic unix/linux # os=Default is meant to be generic unix/linux
known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armhf Linux-ppc Linux-ppc64 Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc64 known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armhf Linux-ppc Linux-ppc64 Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64
os_arch := $(OS_NAME)-$(OS_ARCH) os_arch := $(OS_NAME)-$(OS_ARCH)
IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7")) IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7"))
@ -116,6 +116,17 @@ Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64_LIBNAME := libsnappyjava.so Linux-ppc64_LIBNAME := libsnappyjava.so
Linux-ppc64_SNAPPY_FLAGS := Linux-ppc64_SNAPPY_FLAGS :=
AIX-ppc_CXX := g++
AIX-ppc_STRIP := strip
AIX-ppc_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
AIX-ppc_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
else
AIX-ppc_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32
endif
AIX-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc_SNAPPY_FLAGS :=
AIX-ppc64_CXX := g++ AIX-ppc64_CXX := g++
AIX-ppc64_STRIP := strip -X64 AIX-ppc64_STRIP := strip -X64
AIX-ppc64_LIBNAME := libsnappyjava.a AIX-ppc64_LIBNAME := libsnappyjava.a
@ -127,6 +138,28 @@ endif
AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc64_SNAPPY_FLAGS := AIX-ppc64_SNAPPY_FLAGS :=
Linux-s390_CXX := g++
Linux-s390_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-s390_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31
else
Linux-s390_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31
endif
Linux-s390_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390_LIBNAME := libsnappyjava.so
Linux-s390_SNAPPY_FLAGS :=
Linux-s390x_CXX := g++
Linux-s390x_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-s390x_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
Linux-s390x_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64
endif
Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390x_LIBNAME := libsnappyjava.so
Linux-s390x_SNAPPY_FLAGS :=
SunOS-x86_CXX := g++ SunOS-x86_CXX := g++
SunOS-x86_STRIP := strip SunOS-x86_STRIP := strip
SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden

12
Milestone.md
View File

@ -1,5 +1,17 @@
Since vesion 1.1.0.x, Java 6 (1.6) or higher is required. Since vesion 1.1.0.x, Java 6 (1.6) or higher is required.
## snappy-java-1.1.2.1 (2016-01-22)
* Fixed #131
## snappy-java-1.1.2 (22 September 2015)
* This is a backward compatible release for 1.1.x.
* Add AIX (32-bit) support.
* There is no upgrade for the native libraries of the other platforms.
* A major change since 1.1.1 is a support for reading concatenated results of SnappyOutputStream(s)
## snappy-java-1.1.2-RC2 (18 May 2015)
* Fix #107: SnappyOutputStream.close() is not idempotent
## snappy-java-1.1.2-RC1 (13 May 2015) ## snappy-java-1.1.2-RC1 (13 May 2015)
* SnappyInputStream now supports reading concatenated compressed results of SnappyOutputStream * SnappyInputStream now supports reading concatenated compressed results of SnappyOutputStream
* There has been no compressed format change since 1.0.5.x. So You can read the compressed results interchangeablly between these versions. * There has been no compressed format change since 1.0.5.x. So You can read the compressed results interchangeablly between these versions.

30
README.md
View File

@ -9,7 +9,7 @@ The snappy-java is a Java port of the snappy
* Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.) * Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
* Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`). * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`. * Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
* [Framing-format support](http://snappy.googlecode.com/svn/trunk/framing_format.txt) (Since 1.1.0 version) * [Framing-format support](https://github.com/google/snappy/blob/master/framing_format.txt) (Since 1.1.0 version)
* OSGi support * OSGi support
* [Apache License Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). Free for both commercial and non-commercial use. * [Apache License Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). Free for both commercial and non-commercial use.
@ -39,7 +39,7 @@ Add the following dependency to your pom.xml:
<dependency> <dependency>
<groupId>org.xerial.snappy</groupId> <groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId> <artifactId>snappy-java</artifactId>
<version>1.1.2-RC1</version> <version>1.1.2.1</version>
<type>jar</type> <type>jar</type>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
@ -47,7 +47,7 @@ Add the following dependency to your pom.xml:
### Using with sbt ### Using with sbt
``` ```
libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.2-RC1" libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.2.1"
``` ```
@ -73,9 +73,9 @@ System.out.println(result);
In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used. In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used.
### Stream-based API ### Stream-based API
Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` are also available for reading/writing large data sets. `SnappyFramedOutputStream`/`SnappyFramedInputStream` can be used for the [framing format](https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt). Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` are also available for reading/writing large data sets. `SnappyFramedOutputStream`/`SnappyFramedInputStream` can be used for the [framing format](https://github.com/google/snappy/blob/master/framing_format.txt).
* See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.1.6/snappy-java-1.1.1.6-javadoc.jar/!/index.html) * See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.2.1/snappy-java-1.1.2.1-javadoc.jar/!/index.html)
#### Compatibility Notes #### Compatibility Notes
* `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. Here is the compatibility matrix of data foramt: * `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. Here is the compatibility matrix of data foramt:
@ -99,7 +99,7 @@ If you have snappy-java-(VERSION).jar in the current directory, use `-classpath`
## Public discussion group ## Public discussion group
Post bug reports or feature request to the Issue Tracker: <https://github.com/xerial/snappy-java/issues> Post bug reports or feature request to the Issue Tracker: <https://github.com/xerial/snappy-java/issues>
Public discussion forum is here: <http://groups.google.com/group/xerial?hl=en Xerial Public Discussion Group> Public discussion forum is here: [Xerial Public Discussion Group](http://groups.google.com/group/xerial?hl=en)
## Building from the source code ## Building from the source code
@ -115,7 +115,7 @@ When building on Solaris use
A file `target/snappy-java-$(version).jar` is the product additionally containing the native library built for your platform. A file `target/snappy-java-$(version).jar` is the product additionally containing the native library built for your platform.
## Building linux x86_64 binary ## Building Linux x86\_64 binary
snappy-java tries to static link libstdc++ to increase the availability for various Linux versions. However, standard distributions of 64-bit Linux OS rarely provide libstdc++ compiled with `-fPIC` option. I currently uses custom g++, compiled as follows: snappy-java tries to static link libstdc++ to increase the availability for various Linux versions. However, standard distributions of 64-bit Linux OS rarely provide libstdc++ compiled with `-fPIC` option. I currently uses custom g++, compiled as follows:
@ -135,6 +135,14 @@ $ make install
This g++ build enables static linking of libstdc++. For more infomation on building GCC, see GCC's home page. This g++ build enables static linking of libstdc++. For more infomation on building GCC, see GCC's home page.
## Building Linux s390/s390x binaries
Older snapshots of snappy contain a buggy config.h.in that does not work properly on some big-endian platforms like Linux on IBM z (s390/s390x). Building snappy-java on s390/s390x requires fetching the snappy source from GitHub, and processing the source with autoconf to obtain a usable config.h. On a RHEL s390x system, these steps produced a working 64-bit snappy-java build (the process should be similar for other distributions):
$ sudo yum install java-1.7.1-ibm-devel libstdc++-static-devel
$ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x
$ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1
## Cross-compiling for other platforms ## Cross-compiling for other platforms
The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM: The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM:
@ -165,6 +173,14 @@ snappy-java uses sbt (simple build tool for Scala) as a build tool. Here is a si
> ~test-only * # run tests that matches a given name pattern > ~test-only * # run tests that matches a given name pattern
> publishM2 # publish jar to $HOME/.m2/repository > publishM2 # publish jar to $HOME/.m2/repository
> package # create jar file > package # create jar file
> findbugs # Produce findbugs report in target/findbugs
> jacoco:cover # Report the code coverage of tests to target/jacoco folder
If you need to see detailed debug messages, launch sbt with `-Dloglevel=debug` option:
```
$ ./sbt -Dloglevel=debug
```
For the details of sbt usage, see my blog post: [Building Java Projects with sbt](http://xerial.org/blog/2014/03/24/sbt/) For the details of sbt usage, see my blog post: [Building Java Projects with sbt](http://xerial.org/blog/2014/03/24/sbt/)

22
build.sbt
View File

@ -1,3 +1,4 @@
import de.johoop.findbugs4sbt.ReportType
name := "snappy-java" name := "snappy-java"
@ -10,7 +11,7 @@ description := "snappy-java: A fast compression/decompression library"
sonatypeProfileName := "org.xerial" sonatypeProfileName := "org.xerial"
pomExtra := { pomExtra := {
<url>https://github.comm/xerial/snappy-java</url> <url>https://github.com/xerial/snappy-java</url>
<licenses> <licenses>
<license> <license>
<name>The Apache Software License, Version 2.0</name> <name>The Apache Software License, Version 2.0</name>
@ -68,6 +69,14 @@ logBuffered in Test := false
incOptions := incOptions.value.withNameHashing(true) incOptions := incOptions.value.withNameHashing(true)
findbugsSettings
findbugsReportType := Some(ReportType.FancyHtml)
findbugsReportPath := Some(crossTarget.value / "findbugs" / "report.html")
jacoco.settings
libraryDependencies ++= Seq( libraryDependencies ++= Seq(
"junit" % "junit" % "4.8.2" % "test", "junit" % "junit" % "4.8.2" % "test",
"org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test", "org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test",
@ -92,15 +101,21 @@ OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""")
OsgiKeys.additionalHeaders := Map( OsgiKeys.additionalHeaders := Map(
"Bundle-NativeCode" -> Seq( "Bundle-NativeCode" -> Seq(
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64", "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64",
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64",
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64",
"org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86", "org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
"org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86", "org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
"org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64", "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64", "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64",
"org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86", "org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
"org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64", "org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64",
"org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm", "org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
"org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64", "org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64",
"org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so;osname=linux;processor=ppc64le", "org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so;osname=linux;processor=ppc64le",
"org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x",
"org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc",
"org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64", "org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
"org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86", "org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
"org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64", "org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
@ -131,3 +146,8 @@ releaseProcess := Seq[ReleaseStep](
ReleaseStep(action = Command.process("sonatypeReleaseAll", _)), ReleaseStep(action = Command.process("sonatypeReleaseAll", _)),
pushChanges pushChanges
) )
com.etsy.sbt.Checkstyle.checkstyleSettings
com.etsy.sbt.Checkstyle.CheckstyleTasks.checkstyleConfig := file("src/checkstyle/checks.xml")

37
lib/inc_linux/jni.h
View File

@ -1,26 +1,26 @@
/* /*
* Copyright (c) 1996, 2006, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
* *
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
* *
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
* *
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
* *
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/ */
/* /*
@ -1951,6 +1951,7 @@ JNI_OnUnload(JavaVM *vm, void *reserved);
#define JNI_VERSION_1_2 0x00010002 #define JNI_VERSION_1_2 0x00010002
#define JNI_VERSION_1_4 0x00010004 #define JNI_VERSION_1_4 0x00010004
#define JNI_VERSION_1_6 0x00010006 #define JNI_VERSION_1_6 0x00010006
#define JNI_VERSION_1_8 0x00010008
#ifdef __cplusplus #ifdef __cplusplus
} /* extern "C" */ } /* extern "C" */

2
project/build.properties
View File

@ -1,2 +1,2 @@
sbt.version=0.13.8 sbt.version=0.13.9

6
project/plugins.sbt
View File

@ -1,12 +1,14 @@
addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.5.0") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.3.0") addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.4.0")
addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.5") addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.5")
addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.7.0") addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.7.0")
addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "0.4.3")

364
sbt
View File

@ -1,36 +1,25 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# #
# A more capable sbt runner, coincidentally also called sbt. # A more capable sbt runner, coincidentally also called sbt.
# Author: Paul Phillips <paulp@typesafe.com> # Author: Paul Phillips <paulp@improving.org>
# todo - make this dynamic # todo - make this dynamic
declare -r sbt_release_version="0.13.8" declare -r sbt_release_version="0.13.8"
declare -r sbt_unreleased_version="0.13.8-SNAPSHOT" # -sbt-dev doesn't work at present declare -r sbt_unreleased_version="0.13.9-M1"
declare -r buildProps="project/build.properties" declare -r buildProps="project/build.properties"
declare sbt_jar sbt_dir sbt_create sbt_launch_dir declare sbt_jar sbt_dir sbt_create sbt_version
declare scala_version java_home sbt_explicit_version declare scala_version sbt_explicit_version
declare verbose debug quiet noshare batch trace_level log_level declare verbose noshare batch trace_level log_level
declare sbt_saved_stty declare sbt_saved_stty debugUs
echoerr () { [[ -z "$quiet" ]] && echo "$@" >&2; } echoerr () { echo >&2 "$@"; }
vlog () { [[ -n "$verbose$debug" ]] && echoerr "$@"; } vlog () { [[ -n "$verbose" ]] && echoerr "$@"; }
dlog () { [[ -n "$debug" ]] && echoerr "$@"; }
# we'd like these set before we get around to properly processing arguments
for arg in "$@"; do
case "$arg" in
-q|-quiet) quiet=true ;;
-d|-debug) debug=true ;;
-v|-verbose) verbose=true ;;
*) ;;
esac
done
# spaces are possible, e.g. sbt.version = 0.13.0 # spaces are possible, e.g. sbt.version = 0.13.0
build_props_sbt () { build_props_sbt () {
[[ -r "$buildProps" ]] && \ [[ -r "$buildProps" ]] && \
grep '^sbt\.version' "$buildProps" | tr '=' ' ' | awk '{ print $2; }' grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
} }
update_build_props_sbt () { update_build_props_sbt () {
@ -41,31 +30,24 @@ update_build_props_sbt () {
perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps" perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps" grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
echoerr "!!!" vlog "!!!"
echoerr "!!! Updated file $buildProps setting sbt.version to: $ver" vlog "!!! Updated file $buildProps setting sbt.version to: $ver"
echoerr "!!! Previous value was: $old" vlog "!!! Previous value was: $old"
echoerr "!!!" vlog "!!!"
} }
} }
sbt_version () { set_sbt_version () {
if [[ -n "$sbt_explicit_version" ]]; then sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
echo "$sbt_explicit_version" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
else export sbt_version
local v="$(build_props_sbt)"
if [[ -n "$v" ]]; then
echo "$v"
else
echo "$sbt_release_version"
fi
fi
} }
# restore stty settings (echo in particular) # restore stty settings (echo in particular)
onSbtRunnerExit() { onSbtRunnerExit() {
[[ -n "$sbt_saved_stty" ]] || return [[ -n "$sbt_saved_stty" ]] || return
dlog "" vlog ""
dlog "restoring stty: $sbt_saved_stty" vlog "restoring stty: $sbt_saved_stty"
stty "$sbt_saved_stty" stty "$sbt_saved_stty"
unset sbt_saved_stty unset sbt_saved_stty
} }
@ -73,7 +55,7 @@ onSbtRunnerExit() {
# save stty and trap exit, to ensure echo is reenabled if we are interrupted. # save stty and trap exit, to ensure echo is reenabled if we are interrupted.
trap onSbtRunnerExit EXIT trap onSbtRunnerExit EXIT
sbt_saved_stty="$(stty -g 2>/dev/null)" sbt_saved_stty="$(stty -g 2>/dev/null)"
dlog "Saved stty: $sbt_saved_stty" vlog "Saved stty: $sbt_saved_stty"
# this seems to cover the bases on OSX, and someone will # this seems to cover the bases on OSX, and someone will
# have to tell me about the others. # have to tell me about the others.
@ -119,12 +101,12 @@ init_default_option_file () {
declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation" declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
declare -r default_jvm_opts="-ea -Dfile.encoding=UTF8 -Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts" declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts"
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
declare -r latest_28="2.8.2" declare -r latest_28="2.8.2"
declare -r latest_29="2.9.3" declare -r latest_29="2.9.3"
declare -r latest_210="2.10.5" declare -r latest_210="2.10.5"
declare -r latest_211="2.11.6" declare -r latest_211="2.11.7"
declare -r script_path="$(get_script_path "$BASH_SOURCE")" declare -r script_path="$(get_script_path "$BASH_SOURCE")"
declare -r script_name="${script_path##*/}" declare -r script_name="${script_path##*/}"
@ -133,7 +115,7 @@ declare -r script_name="${script_path##*/}"
declare java_cmd="java" declare java_cmd="java"
declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
declare sbt_launch_repo="http://typesafe.artifactoryonline.com/typesafe/ivy-releases" declare sbt_launch_repo="http://repo.typesafe.com/typesafe/ivy-releases"
# pull -J and -D options to give to java. # pull -J and -D options to give to java.
declare -a residual_args declare -a residual_args
@ -144,14 +126,79 @@ declare -a sbt_commands
# args to jvm/sbt via files or environment variables # args to jvm/sbt via files or environment variables
declare -a extra_jvm_opts extra_sbt_opts declare -a extra_jvm_opts extra_sbt_opts
# if set, use JAVA_HOME over java found in path addJava () {
[[ -e "$JAVA_HOME/bin/java" ]] && java_cmd="$JAVA_HOME/bin/java" vlog "[addJava] arg = '$1'"
java_args+=("$1")
}
addSbt () {
vlog "[addSbt] arg = '$1'"
sbt_commands+=("$1")
}
setThisBuild () {
vlog "[addBuild] args = '$@'"
local key="$1" && shift
addSbt "set $key in ThisBuild := $@"
}
addScalac () {
vlog "[addScalac] arg = '$1'"
scalac_args+=("$1")
}
addResidual () {
vlog "[residual] arg = '$1'"
residual_args+=("$1")
}
addResolver () {
addSbt "set resolvers += $1"
}
addDebugger () {
addJava "-Xdebug"
addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
}
setScalaVersion () {
[[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
addSbt "++ $1"
}
setJavaHome () {
java_cmd="$1/bin/java"
setThisBuild javaHome "Some(file(\"$1\"))"
export JAVA_HOME="$1"
export JDK_HOME="$1"
export PATH="$JAVA_HOME/bin:$PATH"
}
setJavaHomeQuietly () {
addSbt warn
setJavaHome "$1"
addSbt info
}
# if set, use JDK_HOME/JAVA_HOME over java found in path
if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then
setJavaHomeQuietly "$JDK_HOME"
elif [[ -e "$JAVA_HOME/bin/java" ]]; then
setJavaHomeQuietly "$JAVA_HOME"
fi
# directory to store sbt launchers # directory to store sbt launchers
declare sbt_launch_dir="$HOME/.sbt/launchers" declare sbt_launch_dir="$HOME/.sbt/launchers"
[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir" [[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)" [[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
java_version () {
local version=$("$java_cmd" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \")
vlog "Detected Java version: $version"
echo "${version:2:1}"
}
# MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+
default_jvm_opts () {
local v="$(java_version)"
if [[ $v -ge 8 ]]; then
echo "$default_jvm_opts_common"
else
echo "-XX:MaxPermSize=384m $default_jvm_opts_common"
fi
}
build_props_scala () { build_props_scala () {
if [[ -r "$buildProps" ]]; then if [[ -r "$buildProps" ]]; then
versionLine="$(grep '^build.scala.versions' "$buildProps")" versionLine="$(grep '^build.scala.versions' "$buildProps")"
@ -162,22 +209,20 @@ build_props_scala () {
execRunner () { execRunner () {
# print the arguments one to a line, quoting any containing spaces # print the arguments one to a line, quoting any containing spaces
[[ "$verbose" || "$debug" ]] && echo "# Executing command line:" && { vlog "# Executing command line:" && {
for arg; do for arg; do
if [[ -n "$arg" ]]; then if [[ -n "$arg" ]]; then
if printf "%s\n" "$arg" | grep -q ' '; then if printf "%s\n" "$arg" | grep -q ' '; then
printf "\"%s\"\n" "$arg" printf >&2 "\"%s\"\n" "$arg"
else else
printf "%s\n" "$arg" printf >&2 "%s\n" "$arg"
fi fi
fi fi
done done
echo "" vlog ""
} }
if [[ -n "$batch" ]]; then [[ -n "$batch" ]] && exec </dev/null
exec </dev/null
fi
exec "$@" exec "$@"
} }
@ -193,13 +238,13 @@ download_url () {
local url="$1" local url="$1"
local jar="$2" local jar="$2"
echo "Downloading sbt launcher for $(sbt_version):" echoerr "Downloading sbt launcher for $sbt_version:"
echo " From $url" echoerr " From $url"
echo " To $jar" echoerr " To $jar"
mkdir -p "${jar%/*}" && { mkdir -p "${jar%/*}" && {
if which curl >/dev/null; then if which curl >/dev/null; then
curl --fail --silent "$url" --output "$jar" curl --fail --silent --location "$url" --output "$jar"
elif which wget >/dev/null; then elif which wget >/dev/null; then
wget --quiet -O "$jar" "$url" wget --quiet -O "$jar" "$url"
fi fi
@ -207,9 +252,8 @@ download_url () {
} }
acquire_sbt_jar () { acquire_sbt_jar () {
for_sbt_version="$(sbt_version)" sbt_url="$(jar_url "$sbt_version")"
sbt_url="$(jar_url "$for_sbt_version")" sbt_jar="$(jar_file "$sbt_version")"
sbt_jar="$(jar_file "$for_sbt_version")"
[[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar" [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
} }
@ -218,11 +262,23 @@ usage () {
cat <<EOM cat <<EOM
Usage: $script_name [options] Usage: $script_name [options]
Note that options which are passed along to sbt begin with -- whereas
options to this runner use a single dash. Any sbt command can be scheduled
to run first by prefixing the command with --, so --warn, --error and so on
are not special.
Output filtering: if there is a file in the home directory called .sbtignore
and this is not an interactive sbt session, the file is treated as a list of
bash regular expressions. Output lines which match any regex are not echoed.
One can see exactly which lines would have been suppressed by starting this
runner with the -x option.
-h | -help print this message -h | -help print this message
-v | -verbose this runner is chattier -v verbose operation (this runner is chattier)
-d | -debug set sbt log level to Debug -d, -w, -q aliases for --debug, --warn, --error (q means quiet)
-q | -quiet set sbt log level to Error -x debug this script
-trace <level> display stack traces with a max of <level> frames (default: -1, traces suppressed) -trace <level> display stack traces with a max of <level> frames (default: -1, traces suppressed)
-debug-inc enable debugging log for the incremental compiler
-no-colors disable ANSI color codes -no-colors disable ANSI color codes
-sbt-create start sbt even if current directory contains no sbt project -sbt-create start sbt even if current directory contains no sbt project
-sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt/<version>) -sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt/<version>)
@ -234,12 +290,12 @@ Usage: $script_name [options]
-batch Disable interactive mode -batch Disable interactive mode
-prompt <expr> Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted -prompt <expr> Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
# sbt version (default: from $buildProps if present, else latest release) # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
!!! The only way to accomplish this pre-0.12.0 if there is a build.properties file which -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version
!!! contains an sbt.version property is to update the file on disk. That's what this does.
-sbt-version <version> use the specified version of sbt (default: $sbt_release_version) -sbt-version <version> use the specified version of sbt (default: $sbt_release_version)
-sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version
-sbt-jar <path> use the specified jar as the sbt launcher -sbt-jar <path> use the specified jar as the sbt launcher
-sbt-launch-dir <path> directory to hold sbt launchers (default: $sbt_launch_dir) -sbt-launch-dir <path> directory to hold sbt launchers (default: ~/.sbt/launchers)
-sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $sbt_launch_repo) -sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $sbt_launch_repo)
# scala version (default: as chosen by sbt) # scala version (default: as chosen by sbt)
@ -256,7 +312,7 @@ Usage: $script_name [options]
# passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
# The default set is used if JVM_OPTS is unset and no -jvm-opts file is found # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
<default> $default_jvm_opts <default> $(default_jvm_opts)
JVM_OPTS environment variable holding either the jvm args directly, or JVM_OPTS environment variable holding either the jvm args directly, or
the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
@ -273,34 +329,6 @@ Usage: $script_name [options]
EOM EOM
} }
addJava () {
dlog "[addJava] arg = '$1'"
java_args=( "${java_args[@]}" "$1" )
}
addSbt () {
dlog "[addSbt] arg = '$1'"
sbt_commands=( "${sbt_commands[@]}" "$1" )
}
addScalac () {
dlog "[addScalac] arg = '$1'"
scalac_args=( "${scalac_args[@]}" "$1" )
}
addResidual () {
dlog "[residual] arg = '$1'"
residual_args=( "${residual_args[@]}" "$1" )
}
addResolver () {
addSbt "set resolvers += $1"
}
addDebugger () {
addJava "-Xdebug"
addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
}
setScalaVersion () {
[[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
addSbt "++ $1"
}
process_args () process_args ()
{ {
require_arg () { require_arg () {
@ -314,45 +342,50 @@ process_args ()
} }
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
case "$1" in case "$1" in
-h|-help) usage; exit 1 ;; -h|-help) usage; exit 1 ;;
-v|-verbose) verbose=true && log_level=Info && shift ;; -v) verbose=true && shift ;;
-d|-debug) debug=true && log_level=Debug && shift ;; -d) addSbt "--debug" && addSbt debug && shift ;;
-q|-quiet) quiet=true && log_level=Error && shift ;; -w) addSbt "--warn" && addSbt warn && shift ;;
-q) addSbt "--error" && addSbt error && shift ;;
-x) debugUs=true && shift ;;
-trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
-no-share) noshare=true && shift ;;
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
-sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
-offline) addSbt "set offline := true" && shift ;;
-jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
-batch) batch=true && shift ;;
-prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
-trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; -sbt-create) sbt_create=true && shift ;;
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
-no-share) noshare=true && shift ;; -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;;
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;;
-sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
-offline) addSbt "set offline := true" && shift ;; -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
-jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
-batch) batch=true && shift ;; -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "Some(file(\"$2\"))" && shift 2 ;;
-prompt) require_arg "expr" "$1" "$2" && addSbt "set shellPrompt in ThisBuild := (s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
-sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
-jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
-sbt-create) sbt_create=true && shift ;; -D*) addJava "$1" && shift ;;
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; -J*) addJava "${1:2}" && shift ;;
-sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; -S*) addScalac "${1:2}" && shift ;;
-sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; -28) setScalaVersion "$latest_28" && shift ;;
-sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; -29) setScalaVersion "$latest_29" && shift ;;
-sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; -210) setScalaVersion "$latest_210" && shift ;;
-scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; -211) setScalaVersion "$latest_211" && shift ;;
-binary-version) require_arg version "$1" "$2" && addSbt "set scalaBinaryVersion in ThisBuild := \"$2\"" && shift 2 ;;
-scala-home) require_arg path "$1" "$2" && addSbt "set every scalaHome := Some(file(\"$2\"))" && shift 2 ;;
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
-sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
-jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
-D*) addJava "$1" && shift ;; --debug) addSbt debug && addResidual "$1" && shift ;;
-J*) addJava "${1:2}" && shift ;; --warn) addSbt warn && addResidual "$1" && shift ;;
-S*) addScalac "${1:2}" && shift ;; --error) addSbt error && addResidual "$1" && shift ;;
-28) setScalaVersion "$latest_28" && shift ;; *) addResidual "$1" && shift ;;
-29) setScalaVersion "$latest_29" && shift ;;
-210) setScalaVersion "$latest_210" && shift ;;
-211) setScalaVersion "$latest_211" && shift ;;
*) addResidual "$1" && shift ;;
esac esac
done done
} }
@ -379,21 +412,20 @@ else
vlog "No extra sbt options have been defined" vlog "No extra sbt options have been defined"
fi fi
[[ -n "$extra_sbt_opts" ]] && process_args "${extra_sbt_opts[@]}" [[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}"
# reset "$@" to the residual args # reset "$@" to the residual args
set -- "${residual_args[@]}" set -- "${residual_args[@]}"
argumentCount=$# argumentCount=$#
# set sbt version
set_sbt_version
# only exists in 0.12+ # only exists in 0.12+
setTraceLevel() { setTraceLevel() {
case "$(sbt_version)" in case "$sbt_version" in
"0.7."* | "0.10."* | "0.11."* ) "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;;
echoerr "Cannot set trace level in sbt version $(sbt_version)" *) setThisBuild traceLevel $trace_level ;;
;;
*)
addSbt "set every traceLevel := $trace_level"
;;
esac esac
} }
@ -402,9 +434,9 @@ setTraceLevel() {
# Update build.properties on disk to set explicit version - sbt gives us no choice # Update build.properties on disk to set explicit version - sbt gives us no choice
[[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version" [[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
vlog "Detected sbt version $(sbt_version)" vlog "Detected sbt version $sbt_version"
[[ -n "$scala_version" ]] && echoerr "Overriding scala version to $scala_version" [[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version"
# no args - alert them there's stuff in here # no args - alert them there's stuff in here
(( argumentCount > 0 )) || { (( argumentCount > 0 )) || {
@ -438,10 +470,10 @@ if [[ -n "$noshare" ]]; then
addJava "$opt" addJava "$opt"
done done
else else
case "$(sbt_version)" in case "$sbt_version" in
"0.7."* | "0.10."* | "0.11."* | "0.12."* ) "0.7."* | "0.10."* | "0.11."* | "0.12."* )
[[ -n "$sbt_dir" ]] || { [[ -n "$sbt_dir" ]] || {
sbt_dir="$HOME/.sbt/$(sbt_version)" sbt_dir="$HOME/.sbt/$sbt_version"
vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." vlog "Using $sbt_dir as sbt dir, -sbt-dir to override."
} }
;; ;;
@ -460,22 +492,52 @@ elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then
extra_jvm_opts=( $JVM_OPTS ) extra_jvm_opts=( $JVM_OPTS )
else else
vlog "Using default jvm options" vlog "Using default jvm options"
extra_jvm_opts=( $default_jvm_opts ) extra_jvm_opts=( $(default_jvm_opts) )
fi fi
# traceLevel is 0.12+ # traceLevel is 0.12+
[[ -n "$trace_level" ]] && setTraceLevel [[ -n "$trace_level" ]] && setTraceLevel
main () {
execRunner "$java_cmd" \
"${extra_jvm_opts[@]}" \
"${java_args[@]}" \
-jar "$sbt_jar" \
"${sbt_commands[@]}" \
"${residual_args[@]}"
}
if [[ -n "$log_level" ]] && [[ "$log_level" != Info ]]; then # sbt inserts this string on certain lines when formatting is enabled:
sbt_commands=("set logLevel in Global := Level.$log_level" "${sbt_commands[@]}") # val OverwriteLine = "\r\u001BM\u001B[2K"
fi # ...in order not to spam the console with a million "Resolving" lines.
# Unfortunately that makes it that much harder to work with when
# we're not going to print those lines anyway. We strip that bit of
# line noise, but leave the other codes to preserve color.
mainFiltered () {
local ansiOverwrite='\r\x1BM\x1B[2K'
local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -)
echoLine () {
local line="$1"
local line1="$(echo "$line" | sed -r 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code.
local line2="$(echo "$line1" | sed -r 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this.
if [[ $line2 =~ $excludeRegex ]]; then
[[ -n $debugUs ]] && echo "[X] $line1"
else
[[ -n $debugUs ]] && echo " $line1" || echo "$line1"
fi
}
echoLine "Starting sbt with output filtering enabled."
main | while read -r line; do echoLine "$line"; done
}
# Only filter if there's a filter file and we don't see a known interactive command.
# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether
# stdin is a terminal is useless because most of my use cases for this filtering are
# exactly when I'm at a terminal, running sbt non-interactively.
shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; }
# run sbt # run sbt
execRunner "$java_cmd" \ if shouldFilter; then mainFiltered; else main; fi
"${extra_jvm_opts[@]}" \
"${java_args[@]}" \
-jar "$sbt_jar" \
"${sbt_commands[@]}" \
"${residual_args[@]}"

117
src/checkstyle/checks.xml Normal file
View File

@ -0,0 +1,117 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="FileTabCharacter"/>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\r"/>
<property name="message" value="Line contains carriage return"/>
</module>
<module name="RegexpMultiline">
<property name="format" value=" \n"/>
<property name="message" value="Line has trailing whitespace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\{\n\n"/>
<property name="message" value="Blank line after opening brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\}"/>
<property name="message" value="Blank line before closing brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\n"/>
<property name="message" value="Multiple consecutive blank lines"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\Z"/>
<property name="message" value="Blank line before end of file"/>
</module>
<module name="TreeWalker">
<module name="EmptyBlock">
<property name="option" value="text"/>
<property name="tokens" value="
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
</module>
<module name="EmptyStatement"/>
<module name="EmptyForInitializerPad"/>
<module name="EmptyForIteratorPad">
<property name="option" value="space"/>
</module>
<module name="MethodParamPad">
<property name="allowLineBreaks" value="true"/>
<property name="option" value="nospace"/>
</module>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
</module>
<module name="LeftCurly">
<property name="option" value="eol"/>
<property name="tokens" value="
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
</module>
<module name="RightCurly">
<property name="option" value="alone"/>
</module>
<module name="GenericWhitespace"/>
<module name="WhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="UpperEll"/>
<module name="DefaultComesLast"/>
<module name="ArrayTypeStyle"/>
<module name="MultipleVariableDeclarations"/>
<module name="ModifierOrder"/>
<module name="OneStatementPerLine"/>
<module name="StringLiteralEquality"/>
<module name="MutableException"/>
<module name="EqualsHashCode"/>
<module name="InnerAssignment"/>
<module name="InterfaceIsType"/>
<module name="HideUtilityClassConstructor"/>
<module name="MemberName"/>
<module name="LocalVariableName"/>
<module name="LocalFinalVariableName"/>
<module name="TypeName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="ClassTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="AvoidStarImport"/>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="ignoreEnhancedForColon" value="false"/>
<property name="tokens" value="
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
</module>
</module>
</module>

59
src/main/java/org/xerial/snappy/OSInfo.java
View File

@ -32,7 +32,6 @@ import java.util.Locale;
* Provides OS name and architecture name. * Provides OS name and architecture name.
* *
* @author leo * @author leo
*
*/ */
public class OSInfo public class OSInfo
{ {
@ -44,6 +43,8 @@ public class OSInfo
public static final String IA64 = "ia64"; public static final String IA64 = "ia64";
public static final String PPC = "ppc"; public static final String PPC = "ppc";
public static final String PPC64 = "ppc64"; public static final String PPC64 = "ppc64";
public static final String IBMZ = "s390";
public static final String IBMZ_64 = "s390x";
static { static {
// x86 mappings // x86 mappings
@ -76,15 +77,21 @@ public class OSInfo
archMapping.put("power_rs", PPC); archMapping.put("power_rs", PPC);
// TODO: PowerPC 64bit mappings // TODO: PowerPC 64bit mappings
archMapping.put(PPC64, PPC64); archMapping.put(PPC64, PPC64);
archMapping.put("power64", PPC64); archMapping.put("power64", PPC64);
archMapping.put("powerpc64", PPC64); archMapping.put("powerpc64", PPC64);
archMapping.put("power_pc64", PPC64); archMapping.put("power_pc64", PPC64);
archMapping.put("power_rs64", PPC64); archMapping.put("power_rs64", PPC64);
// IBM z mappings
archMapping.put(IBMZ, IBMZ);
// IBM z 64-bit mappings
archMapping.put(IBMZ_64, IBMZ_64);
} }
public static void main(String[] args)
public static void main(String[] args) { {
if (args.length >= 1) { if (args.length >= 1) {
if ("--os".equals(args[0])) { if ("--os".equals(args[0])) {
System.out.print(getOSName()); System.out.print(getOSName());
@ -99,27 +106,31 @@ public class OSInfo
System.out.print(getNativeLibFolderPathForCurrentOS()); System.out.print(getNativeLibFolderPathForCurrentOS());
} }
public static String getNativeLibFolderPathForCurrentOS() { public static String getNativeLibFolderPathForCurrentOS()
{
return getOSName() + "/" + getArchName(); return getOSName() + "/" + getArchName();
} }
public static String getOSName() { public static String getOSName()
{
return translateOSNameToFolderName(System.getProperty("os.name")); return translateOSNameToFolderName(System.getProperty("os.name"));
} }
public static String getArchName() { public static String getArchName()
{
// if running Linux on ARM, need to determine ABI of JVM // if running Linux on ARM, need to determine ABI of JVM
String osArch = System.getProperty("os.arch"); String osArch = System.getProperty("os.arch");
if (osArch.startsWith("arm") && System.getProperty("os.name").contains("Linux")) { if (osArch.startsWith("arm") && System.getProperty("os.name").contains("Linux")) {
String javaHome = System.getProperty("java.home"); String javaHome = System.getProperty("java.home");
try { try {
// determine if first JVM found uses ARM hard-float ABI // determine if first JVM found uses ARM hard-float ABI
String[] cmdarray = { "/bin/sh", "-c", "find '" + javaHome + String[] cmdarray = {"/bin/sh", "-c", "find '" + javaHome +
"' -name 'libjvm.so' | head -1 | xargs readelf -A | " + "' -name 'libjvm.so' | head -1 | xargs readelf -A | " +
"grep 'Tag_ABI_VFP_args: VFP registers'" }; "grep 'Tag_ABI_VFP_args: VFP registers'"};
int exitCode = Runtime.getRuntime().exec(cmdarray).waitFor(); int exitCode = Runtime.getRuntime().exec(cmdarray).waitFor();
if (exitCode == 0) if (exitCode == 0) {
return "armhf"; return "armhf";
}
} }
catch (IOException e) { catch (IOException e) {
// ignored: fall back to "arm" arch (soft-float ABI) // ignored: fall back to "arm" arch (soft-float ABI)
@ -130,13 +141,15 @@ public class OSInfo
} }
else { else {
String lc = osArch.toLowerCase(Locale.US); String lc = osArch.toLowerCase(Locale.US);
if(archMapping.containsKey(lc)) if (archMapping.containsKey(lc)) {
return archMapping.get(lc); return archMapping.get(lc);
}
} }
return translateArchNameToFolderName(osArch); return translateArchNameToFolderName(osArch);
} }
static String translateOSNameToFolderName(String osName) { static String translateOSNameToFolderName(String osName)
{
if (osName.contains("Windows")) { if (osName.contains("Windows")) {
return "Windows"; return "Windows";
} }
@ -146,16 +159,16 @@ public class OSInfo
else if (osName.contains("Linux")) { else if (osName.contains("Linux")) {
return "Linux"; return "Linux";
} }
else if (osName.contains("AIX")) { else if (osName.contains("AIX")) {
return "AIX"; return "AIX";
} }
else {
else {
return osName.replaceAll("\\W", ""); return osName.replaceAll("\\W", "");
} }
} }
static String translateArchNameToFolderName(String archName) { static String translateArchNameToFolderName(String archName)
{
return archName.replaceAll("\\W", ""); return archName.replaceAll("\\W", "");
} }
} }

1176
src/main/java/org/xerial/snappy/PureJavaCrc32C.java
View File

File diff suppressed because it is too large Load Diff

391
src/main/java/org/xerial/snappy/Snappy.java
View File

@ -25,6 +25,7 @@
package org.xerial.snappy; package org.xerial.snappy;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URL; import java.net.URL;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
@ -33,12 +34,11 @@ import java.util.Properties;
/** /**
* Snappy API for data compression/decompression * Snappy API for data compression/decompression
* * <p/>
* Note: if the native libraries cannot be loaded, an ExceptionInInitializerError * Note: if the native libraries cannot be loaded, an ExceptionInInitializerError
* will be thrown at first use of this class. * will be thrown at first use of this class.
* *
* @author Taro L. Saito * @author Taro L. Saito
*
*/ */
public class Snappy public class Snappy
{ {
@ -56,7 +56,6 @@ public class Snappy
*/ */
private static SnappyNative impl; private static SnappyNative impl;
/** /**
* Clean up a temporary file (native lib) generated by snappy-java. * Clean up a temporary file (native lib) generated by snappy-java.
* General users do not need to call this method, since the native library extracted in snappy-java * General users do not need to call this method, since the native library extracted in snappy-java
@ -64,29 +63,25 @@ public class Snappy
* This method is useful when using a J2EE container, which will restart servlet containers multiple times without * This method is useful when using a J2EE container, which will restart servlet containers multiple times without
* restarting JVM. * restarting JVM.
*/ */
public static void cleanUp() { public static void cleanUp()
{
SnappyLoader.cleanUpExtractedNativeLib(); SnappyLoader.cleanUpExtractedNativeLib();
SnappyLoader.setApi(null); SnappyLoader.setApi(null);
} }
/** /**
* Copy bytes from source to destination * Copy bytes from source to destination
* *
* @param src * @param src pointer to the source array
* pointer to the source array * @param offset byte offset in the source array
* @param offset * @param byteLength the number of bytes to copy
* byte offset in the source array * @param dest pointer to the destination array
* @param byteLength * @param dest_offset byte offset in the destination array
* the number of bytes to copy
* @param dest
* pointer to the destination array
* @param dest_offset
* byte offset in the destination array
* @throws IOException * @throws IOException
*/ */
public static void arrayCopy(Object src, int offset, int byteLength, Object dest, int dest_offset) public static void arrayCopy(Object src, int offset, int byteLength, Object dest, int dest_offset)
throws IOException { throws IOException
{
impl.arrayCopy(src, offset, byteLength, dest, dest_offset); impl.arrayCopy(src, offset, byteLength, dest, dest_offset);
} }
@ -96,12 +91,13 @@ public class Snappy
* cost, use {@link #compress(byte[], int, int, byte[], int)} or * cost, use {@link #compress(byte[], int, int, byte[], int)} or
* {@link #compress(ByteBuffer, ByteBuffer)}. * {@link #compress(ByteBuffer, ByteBuffer)}.
* *
* @param input * @param input the input data
* the input data
* @return the compressed byte array * @return the compressed byte array
* @throws IOException * @throws IOException
*/ */
public static byte[] compress(byte[] input) throws IOException { public static byte[] compress(byte[] input)
throws IOException
{
return rawCompress(input, input.length); return rawCompress(input, input.length);
} }
@ -115,11 +111,11 @@ public class Snappy
* @param output * @param output
* @param outputOffset * @param outputOffset
* @return byte size of the compressed data * @return byte size of the compressed data
* @throws IOException * @throws IOException when failed to access the input/output buffer
* when failed to access the input/output buffer
*/ */
public static int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) public static int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws IOException { throws IOException
{
return rawCompress(input, inputOffset, inputLength, output, outputOffset); return rawCompress(input, inputOffset, inputLength, output, outputOffset);
} }
@ -128,21 +124,21 @@ public class Snappy
* you can retrieve the compressed data from the output buffer [pos() ... * you can retrieve the compressed data from the output buffer [pos() ...
* limit()) (compressed data size = limit() - pos() = remaining()) * limit()) (compressed data size = limit() - pos() = remaining())
* *
* @param uncompressed * @param uncompressed buffer[pos() ... limit()) containing the input data
* buffer[pos() ... limit()) containing the input data * @param compressed output of the compressed data. Uses range [pos()..].
* @param compressed
* output of the compressed data. Uses range [pos()..].
* @return byte size of the compressed data. * @return byte size of the compressed data.
* * @throws SnappyError when the input is not a direct buffer
* @throws SnappyError
* when the input is not a direct buffer
*/ */
public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) throws IOException { public static int compress(ByteBuffer uncompressed, ByteBuffer compressed)
throws IOException
{
if (!uncompressed.isDirect()) if (!uncompressed.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
if (!compressed.isDirect()) }
if (!compressed.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: uncompressed[pos(), limit()) // input: uncompressed[pos(), limit())
// output: compressed // output: compressed
@ -164,7 +160,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(char[] input) throws IOException { public static byte[] compress(char[] input)
throws IOException
{
return rawCompress(input, input.length * 2); // char uses 2 bytes return rawCompress(input, input.length * 2); // char uses 2 bytes
} }
@ -174,7 +172,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(double[] input) throws IOException { public static byte[] compress(double[] input)
throws IOException
{
return rawCompress(input, input.length * 8); // double uses 8 bytes return rawCompress(input, input.length * 8); // double uses 8 bytes
} }
@ -184,7 +184,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(float[] input) throws IOException { public static byte[] compress(float[] input)
throws IOException
{
return rawCompress(input, input.length * 4); // float uses 4 bytes return rawCompress(input, input.length * 4); // float uses 4 bytes
} }
@ -194,7 +196,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(int[] input) throws IOException { public static byte[] compress(int[] input)
throws IOException
{
return rawCompress(input, input.length * 4); // int uses 4 bytes return rawCompress(input, input.length * 4); // int uses 4 bytes
} }
@ -204,7 +208,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(long[] input) throws IOException { public static byte[] compress(long[] input)
throws IOException
{
return rawCompress(input, input.length * 8); // long uses 8 bytes return rawCompress(input, input.length * 8); // long uses 8 bytes
} }
@ -214,7 +220,9 @@ public class Snappy
* @param input * @param input
* @return the compressed data * @return the compressed data
*/ */
public static byte[] compress(short[] input) throws IOException { public static byte[] compress(short[] input)
throws IOException
{
return rawCompress(input, input.length * 2); // short uses 2 bytes return rawCompress(input, input.length * 2); // short uses 2 bytes
} }
@ -225,7 +233,9 @@ public class Snappy
* @return the compressed data * @return the compressed data
* @throws IOException * @throws IOException
*/ */
public static byte[] compress(String s) throws IOException { public static byte[] compress(String s)
throws IOException
{
try { try {
return compress(s, "UTF-8"); return compress(s, "UTF-8");
} }
@ -243,7 +253,9 @@ public class Snappy
* @throws UnsupportedEncodingException * @throws UnsupportedEncodingException
* @throws IOException * @throws IOException
*/ */
public static byte[] compress(String s, String encoding) throws UnsupportedEncodingException, IOException { public static byte[] compress(String s, String encoding)
throws UnsupportedEncodingException, IOException
{
byte[] data = s.getBytes(encoding); byte[] data = s.getBytes(encoding);
return compress(data); return compress(data);
} }
@ -257,7 +269,9 @@ public class Snappy
* @throws UnsupportedEncodingException * @throws UnsupportedEncodingException
* @throws IOException * @throws IOException
*/ */
public static byte[] compress(String s, Charset encoding) throws IOException { public static byte[] compress(String s, Charset encoding)
throws IOException
{
byte[] data = s.getBytes(encoding); byte[] data = s.getBytes(encoding);
return compress(data); return compress(data);
} }
@ -267,19 +281,29 @@ public class Snappy
* *
* @return native library version * @return native library version
*/ */
public static String getNativeLibraryVersion() { public static String getNativeLibraryVersion()
{
URL versionFile = SnappyLoader.class.getResource("/org/xerial/snappy/VERSION"); URL versionFile = SnappyLoader.class.getResource("/org/xerial/snappy/VERSION");
String version = "unknown"; String version = "unknown";
try { try {
if (versionFile != null) { if (versionFile != null) {
Properties versionData = new Properties(); InputStream in = null;
versionData.load(versionFile.openStream()); try {
version = versionData.getProperty("version", version); Properties versionData = new Properties();
if (version.equals("unknown")) in = versionFile.openStream();
version = versionData.getProperty("VERSION", version); versionData.load(in);
version = version.trim().replaceAll("[^0-9\\.]", ""); version = versionData.getProperty("version", version);
if (version.equals("unknown")) {
version = versionData.getProperty("VERSION", version);
}
version = version.trim().replaceAll("[^0-9\\.]", "");
}
finally {
if(in != null) {
in.close();
}
}
} }
} }
catch (IOException e) { catch (IOException e) {
@ -294,9 +318,12 @@ public class Snappy
* uncompressed data. Takes time proportional to the input length, but is * uncompressed data. Takes time proportional to the input length, but is
* usually at least a factor of four faster than actual decompression. * usually at least a factor of four faster than actual decompression.
*/ */
public static boolean isValidCompressedBuffer(byte[] input, int offset, int length) throws IOException { public static boolean isValidCompressedBuffer(byte[] input, int offset, int length)
if (input == null) throws IOException
{
if (input == null) {
throw new NullPointerException("input is null"); throw new NullPointerException("input is null");
}
return impl.isValidCompressedBuffer(input, offset, length); return impl.isValidCompressedBuffer(input, offset, length);
} }
@ -306,7 +333,9 @@ public class Snappy
* uncompressed data. Takes time proportional to the input length, but is * uncompressed data. Takes time proportional to the input length, but is
* usually at least a factor of four faster than actual decompression. * usually at least a factor of four faster than actual decompression.
*/ */
public static boolean isValidCompressedBuffer(byte[] input) throws IOException { public static boolean isValidCompressedBuffer(byte[] input)
throws IOException
{
return isValidCompressedBuffer(input, 0, input.length); return isValidCompressedBuffer(input, 0, input.length);
} }
@ -316,7 +345,9 @@ public class Snappy
* Takes time proportional to the input length, but is usually at least a * Takes time proportional to the input length, but is usually at least a
* factor of four faster than actual decompression. * factor of four faster than actual decompression.
*/ */
public static boolean isValidCompressedBuffer(ByteBuffer compressed) throws IOException { public static boolean isValidCompressedBuffer(ByteBuffer compressed)
throws IOException
{
return impl.isValidCompressedBuffer(compressed, compressed.position(), return impl.isValidCompressedBuffer(compressed, compressed.position(),
compressed.remaining()); compressed.remaining());
} }
@ -327,58 +358,64 @@ public class Snappy
* uncompressed data. Takes time proportional to the input length, but is * uncompressed data. Takes time proportional to the input length, but is
* usually at least a factor of four faster than actual decompression. * usually at least a factor of four faster than actual decompression.
*/ */
public static boolean isValidCompressedBuffer(long inputAddr, long offset, long length) throws IOException { public static boolean isValidCompressedBuffer(long inputAddr, long offset, long length)
throws IOException
{
return impl.isValidCompressedBuffer(inputAddr, offset, length); return impl.isValidCompressedBuffer(inputAddr, offset, length);
} }
/** /**
* Get the maximum byte size needed for compressing data of the given byte * Get the maximum byte size needed for compressing data of the given byte
* size. * size.
* *
* @param byteSize * @param byteSize byte size of the data to compress
* byte size of the data to compress
* @return maximum byte size of the compressed data * @return maximum byte size of the compressed data
*/ */
public static int maxCompressedLength(int byteSize) { public static int maxCompressedLength(int byteSize)
{
return impl.maxCompressedLength(byteSize); return impl.maxCompressedLength(byteSize);
} }
/** /**
* Zero-copy compress using memory addresses. * Zero-copy compress using memory addresses.
*
* @param inputAddr input memory address * @param inputAddr input memory address
* @param inputSize input byte size * @param inputSize input byte size
* @param destAddr destination address of the compressed data * @param destAddr destination address of the compressed data
* @return the compressed data size * @return the compressed data size
* @throws IOException * @throws IOException
*/ */
public static long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException { public static long rawCompress(long inputAddr, long inputSize, long destAddr)
throws IOException
{
return impl.rawCompress(inputAddr, inputSize, destAddr); return impl.rawCompress(inputAddr, inputSize, destAddr);
} }
/** /**
* Zero-copy decompress using memory addresses. * Zero-copy decompress using memory addresses.
*
* @param inputAddr input memory address * @param inputAddr input memory address
* @param inputSize input byte size * @param inputSize input byte size
* @param destAddr destination address of the uncompressed data * @param destAddr destination address of the uncompressed data
* @return the uncompressed data size * @return the uncompressed data size
* @throws IOException * @throws IOException
*/ */
public static long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException { public static long rawUncompress(long inputAddr, long inputSize, long destAddr)
throws IOException
{
return impl.rawUncompress(inputAddr, inputSize, destAddr); return impl.rawUncompress(inputAddr, inputSize, destAddr);
} }
/** /**
* Compress the input data and produce a byte array of the uncompressed data * Compress the input data and produce a byte array of the uncompressed data
* *
* @param data * @param data input array. The input MUST be an array type
* input array. The input MUST be an array type * @param byteSize the input byte size
* @param byteSize
* the input byte size
* @return compressed data * @return compressed data
*/ */
public static byte[] rawCompress(Object data, int byteSize) throws IOException { public static byte[] rawCompress(Object data, int byteSize)
throws IOException
{
byte[] buf = new byte[Snappy.maxCompressedLength(byteSize)]; byte[] buf = new byte[Snappy.maxCompressedLength(byteSize)];
int compressedByteSize = impl.rawCompress(data, 0, byteSize, buf, 0); int compressedByteSize = impl.rawCompress(data, 0, byteSize, buf, 0);
byte[] result = new byte[compressedByteSize]; byte[] result = new byte[compressedByteSize];
@ -390,23 +427,20 @@ public class Snappy
* Compress the input buffer [offset,... ,offset+length) contents, then * Compress the input buffer [offset,... ,offset+length) contents, then
* write the compressed data to the output buffer[offset, ...) * write the compressed data to the output buffer[offset, ...)
* *
* @param input * @param input input array. This MUST be a primitive array type
* input array. This MUST be a primitive array type * @param inputOffset byte offset at the output array
* @param inputOffset * @param inputLength byte length of the input data
* byte offset at the output array * @param output output array. This MUST be a primitive array type
* @param inputLength * @param outputOffset byte offset at the output array
* byte length of the input data
* @param output
* output array. This MUST be a primitive array type
* @param outputOffset
* byte offset at the output array
* @return byte size of the compressed data * @return byte size of the compressed data
* @throws IOException * @throws IOException
*/ */
public static int rawCompress(Object input, int inputOffset, int inputLength, byte[] output, int outputOffset) public static int rawCompress(Object input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws IOException { throws IOException
if (input == null || output == null) {
if (input == null || output == null) {
throw new NullPointerException("input or output is null"); throw new NullPointerException("input or output is null");
}
int compressedSize = impl int compressedSize = impl
.rawCompress(input, inputOffset, inputLength, output, outputOffset); .rawCompress(input, inputOffset, inputLength, output, outputOffset);
@ -416,31 +450,27 @@ public class Snappy
/** /**
* Uncompress the content in the input buffer. The uncompressed data is * Uncompress the content in the input buffer. The uncompressed data is
* written to the output buffer. * written to the output buffer.
* * <p/>
* Note that if you pass the wrong data or the range [inputOffset, * Note that if you pass the wrong data or the range [inputOffset,
* inputOffset + inputLength) that cannot be uncompressed, your JVM might * inputOffset + inputLength) that cannot be uncompressed, your JVM might
* crash due to the access violation exception issued in the native code * crash due to the access violation exception issued in the native code
* written in C++. To avoid this type of crash, use * written in C++. To avoid this type of crash, use
* {@link #isValidCompressedBuffer(byte[], int, int)} first. * {@link #isValidCompressedBuffer(byte[], int, int)} first.
* *
* @param input * @param input input byte array
* input byte array * @param inputOffset byte offset in the input byte array
* @param inputOffset * @param inputLength byte length of the input data
* byte offset in the input byte array * @param output output buffer, MUST be a primitive type array
* @param inputLength * @param outputOffset byte offset in the output buffer
* byte length of the input data
* @param output
* output buffer, MUST be a primitive type array
* @param outputOffset
* byte offset in the output buffer
* @return the byte size of the uncompressed data * @return the byte size of the uncompressed data
* @throws IOException * @throws IOException when failed to uncompress the input data
* when failed to uncompress the input data
*/ */
public static int rawUncompress(byte[] input, int inputOffset, int inputLength, Object output, int outputOffset) public static int rawUncompress(byte[] input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException { throws IOException
if (input == null || output == null) {
if (input == null || output == null) {
throw new NullPointerException("input or output is null"); throw new NullPointerException("input or output is null");
}
return impl.rawUncompress(input, inputOffset, inputLength, output, outputOffset); return impl.rawUncompress(input, inputOffset, inputLength, output, outputOffset);
} }
@ -451,16 +481,18 @@ public class Snappy
* @return the uncompressed byte array * @return the uncompressed byte array
* @throws IOException * @throws IOException
*/ */
public static byte[] uncompress(byte[] input) throws IOException { public static byte[] uncompress(byte[] input)
throws IOException
{
byte[] result = new byte[Snappy.uncompressedLength(input)]; byte[] result = new byte[Snappy.uncompressedLength(input)];
int byteSize = Snappy.uncompress(input, 0, input.length, result, 0); Snappy.uncompress(input, 0, input.length, result, 0);
return result; return result;
} }
/** /**
* Uncompress the content in the input buffer. The uncompressed data is * Uncompress the content in the input buffer. The uncompressed data is
* written to the output buffer. * written to the output buffer.
* * <p/>
* Note that if you pass the wrong data or the range [inputOffset, * Note that if you pass the wrong data or the range [inputOffset,
* inputOffset + inputLength) that cannot be uncompressed, your JVM might * inputOffset + inputLength) that cannot be uncompressed, your JVM might
* crash due to the access violation exception issued in the native code * crash due to the access violation exception issued in the native code
@ -476,37 +508,36 @@ public class Snappy
* @throws IOException * @throws IOException
*/ */
public static int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) public static int uncompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset)
throws IOException { throws IOException
{
return rawUncompress(input, inputOffset, inputLength, output, outputOffset); return rawUncompress(input, inputOffset, inputLength, output, outputOffset);
} }
/** /**
* Uncompress the content in the input buffer. The result is dumped to the * Uncompress the content in the input buffer. The result is dumped to the
* specified output buffer. * specified output buffer.
* * <p/>
* Note that if you pass the wrong data or the range [pos(), limit()) that * Note that if you pass the wrong data or the range [pos(), limit()) that
* cannot be uncompressed, your JVM might crash due to the access violation * cannot be uncompressed, your JVM might crash due to the access violation
* exception issued in the native code written in C++. To avoid this type of * exception issued in the native code written in C++. To avoid this type of
* crash, use {@link #isValidCompressedBuffer(ByteBuffer)} first. * crash, use {@link #isValidCompressedBuffer(ByteBuffer)} first.
* *
* * @param compressed buffer[pos() ... limit()) containing the input data
* @param compressed * @param uncompressed output of the the uncompressed data. It uses buffer[pos()..]
* buffer[pos() ... limit()) containing the input data
* @param uncompressed
* output of the the uncompressed data. It uses buffer[pos()..]
* @return uncompressed data size * @return uncompressed data size
* * @throws IOException when failed to uncompress the given input
* @throws IOException * @throws SnappyError when the input is not a direct buffer
* when failed to uncompress the given input
* @throws SnappyError
* when the input is not a direct buffer
*/ */
public static int uncompress(ByteBuffer compressed, ByteBuffer uncompressed) throws IOException { public static int uncompress(ByteBuffer compressed, ByteBuffer uncompressed)
throws IOException
{
if (!compressed.isDirect()) if (!compressed.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
if (!uncompressed.isDirect()) }
if (!uncompressed.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
int cPos = compressed.position(); int cPos = compressed.position();
int cLen = compressed.remaining(); int cLen = compressed.remaining();
@ -527,7 +558,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static char[] uncompressCharArray(byte[] input) throws IOException { public static char[] uncompressCharArray(byte[] input)
throws IOException
{
return uncompressCharArray(input, 0, input.length); return uncompressCharArray(input, 0, input.length);
} }
@ -540,10 +573,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static char[] uncompressCharArray(byte[] input, int offset, int length) throws IOException { public static char[] uncompressCharArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length); int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
char[] result = new char[uncompressedLength / 2]; char[] result = new char[uncompressedLength / 2];
int byteSize = impl.rawUncompress(input, offset, length, result, 0); impl.rawUncompress(input, offset, length, result, 0);
return result; return result;
} }
@ -554,10 +589,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static double[] uncompressDoubleArray(byte[] input) throws IOException { public static double[] uncompressDoubleArray(byte[] input)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, 0, input.length); int uncompressedLength = Snappy.uncompressedLength(input, 0, input.length);
double[] result = new double[uncompressedLength / 8]; double[] result = new double[uncompressedLength / 8];
int byteSize = impl.rawUncompress(input, 0, input.length, result, 0); impl.rawUncompress(input, 0, input.length, result, 0);
return result; return result;
} }
@ -567,11 +604,12 @@ public class Snappy
* *
* @param input * @param input
* @return uncompressed byte size of the the given input data * @return uncompressed byte size of the the given input data
* @throws IOException * @throws IOException when failed to uncompress the given input. The error code is
* when failed to uncompress the given input. The error code is * {@link SnappyErrorCode#PARSING_ERROR}
* {@link SnappyErrorCode#PARSING_ERROR}
*/ */
public static int uncompressedLength(byte[] input) throws IOException { public static int uncompressedLength(byte[] input)
throws IOException
{
return impl.uncompressedLength(input, 0, input.length); return impl.uncompressedLength(input, 0, input.length);
} }
@ -583,13 +621,15 @@ public class Snappy
* @param offset * @param offset
* @param length * @param length
* @return uncompressed byte size of the the given input data * @return uncompressed byte size of the the given input data
* @throws IOException * @throws IOException when failed to uncompress the given input. The error code is
* when failed to uncompress the given input. The error code is * {@link SnappyErrorCode#PARSING_ERROR}
* {@link SnappyErrorCode#PARSING_ERROR}
*/ */
public static int uncompressedLength(byte[] input, int offset, int length) throws IOException { public static int uncompressedLength(byte[] input, int offset, int length)
if (input == null) throws IOException
{
if (input == null) {
throw new NullPointerException("input is null"); throw new NullPointerException("input is null");
}
return impl.uncompressedLength(input, offset, length); return impl.uncompressedLength(input, offset, length);
} }
@ -598,31 +638,34 @@ public class Snappy
* Get the uncompressed byte size of the given compressed input. This * Get the uncompressed byte size of the given compressed input. This
* operation takes O(1) time. * operation takes O(1) time.
* *
* @param compressed * @param compressed input data [pos() ... limit())
* input data [pos() ... limit())
* @return uncompressed byte length of the given input * @return uncompressed byte length of the given input
* @throws IOException * @throws IOException when failed to uncompress the given input. The error code is
* when failed to uncompress the given input. The error code is * {@link SnappyErrorCode#PARSING_ERROR}
* {@link SnappyErrorCode#PARSING_ERROR} * @throws SnappyError when the input is not a direct buffer
* @throws SnappyError
* when the input is not a direct buffer
*/ */
public static int uncompressedLength(ByteBuffer compressed) throws IOException { public static int uncompressedLength(ByteBuffer compressed)
if (!compressed.isDirect()) throws IOException
{
if (!compressed.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer"); throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
return impl.uncompressedLength(compressed, compressed.position(), compressed.remaining()); return impl.uncompressedLength(compressed, compressed.position(), compressed.remaining());
} }
/** /**
* Get the uncompressed byte size of the given compressed input. This operation takes O(1) time. * Get the uncompressed byte size of the given compressed input. This operation takes O(1) time.
*
* @param inputAddr compressed data address * @param inputAddr compressed data address
* @param len byte length of the input * @param len byte length of the input
* @return uncompressed byte length of the given input * @return uncompressed byte length of the given input
* @throws IOException when failed to uncompress the given input. The error code is * @throws IOException when failed to uncompress the given input. The error code is
* {@link SnappyErrorCode#PARSING_ERROR} * {@link SnappyErrorCode#PARSING_ERROR}
*/ */
public static long uncompressedLength(long inputAddr, long len) throws IOException { public static long uncompressedLength(long inputAddr, long len)
throws IOException
{
return impl.uncompressedLength(inputAddr, len); return impl.uncompressedLength(inputAddr, len);
} }
@ -633,7 +676,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static float[] uncompressFloatArray(byte[] input) throws IOException { public static float[] uncompressFloatArray(byte[] input)
throws IOException
{
return uncompressFloatArray(input, 0, input.length); return uncompressFloatArray(input, 0, input.length);
} }
@ -646,10 +691,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static float[] uncompressFloatArray(byte[] input, int offset, int length) throws IOException { public static float[] uncompressFloatArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length); int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
float[] result = new float[uncompressedLength / 4]; float[] result = new float[uncompressedLength / 4];
int byteSize = impl.rawUncompress(input, offset, length, result, 0); impl.rawUncompress(input, offset, length, result, 0);
return result; return result;
} }
@ -660,7 +707,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static int[] uncompressIntArray(byte[] input) throws IOException { public static int[] uncompressIntArray(byte[] input)
throws IOException
{
return uncompressIntArray(input, 0, input.length); return uncompressIntArray(input, 0, input.length);
} }
@ -673,10 +722,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static int[] uncompressIntArray(byte[] input, int offset, int length) throws IOException { public static int[] uncompressIntArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length); int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
int[] result = new int[uncompressedLength / 4]; int[] result = new int[uncompressedLength / 4];
int byteSize = impl.rawUncompress(input, offset, length, result, 0); impl.rawUncompress(input, offset, length, result, 0);
return result; return result;
} }
@ -687,7 +738,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static long[] uncompressLongArray(byte[] input) throws IOException { public static long[] uncompressLongArray(byte[] input)
throws IOException
{
return uncompressLongArray(input, 0, input.length); return uncompressLongArray(input, 0, input.length);
} }
@ -700,10 +753,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static long[] uncompressLongArray(byte[] input, int offset, int length) throws IOException { public static long[] uncompressLongArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length); int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
long[] result = new long[uncompressedLength / 8]; long[] result = new long[uncompressedLength / 8];
int byteSize = impl.rawUncompress(input, offset, length, result, 0); impl.rawUncompress(input, offset, length, result, 0);
return result; return result;
} }
@ -714,7 +769,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static short[] uncompressShortArray(byte[] input) throws IOException { public static short[] uncompressShortArray(byte[] input)
throws IOException
{
return uncompressShortArray(input, 0, input.length); return uncompressShortArray(input, 0, input.length);
} }
@ -727,10 +784,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static short[] uncompressShortArray(byte[] input, int offset, int length) throws IOException { public static short[] uncompressShortArray(byte[] input, int offset, int length)
throws IOException
{
int uncompressedLength = Snappy.uncompressedLength(input, offset, length); int uncompressedLength = Snappy.uncompressedLength(input, offset, length);
short[] result = new short[uncompressedLength / 2]; short[] result = new short[uncompressedLength / 2];
int byteSize = impl.rawUncompress(input, offset, length, result, 0); impl.rawUncompress(input, offset, length, result, 0);
return result; return result;
} }
@ -741,7 +800,9 @@ public class Snappy
* @return the uncompressed dasta * @return the uncompressed dasta
* @throws IOException * @throws IOException
*/ */
public static String uncompressString(byte[] input) throws IOException { public static String uncompressString(byte[] input)
throws IOException
{
try { try {
return uncompressString(input, "UTF-8"); return uncompressString(input, "UTF-8");
} }
@ -759,7 +820,9 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static String uncompressString(byte[] input, int offset, int length) throws IOException { public static String uncompressString(byte[] input, int offset, int length)
throws IOException
{
try { try {
return uncompressString(input, offset, length, "UTF-8"); return uncompressString(input, offset, length, "UTF-8");
} }
@ -779,10 +842,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static String uncompressString(byte[] input, int offset, int length, String encoding) throws IOException, public static String uncompressString(byte[] input, int offset, int length, String encoding)
UnsupportedEncodingException { throws IOException,
UnsupportedEncodingException
{
byte[] uncompressed = new byte[uncompressedLength(input, offset, length)]; byte[] uncompressed = new byte[uncompressedLength(input, offset, length)];
int compressedSize = uncompress(input, offset, length, uncompressed, 0); uncompress(input, offset, length, uncompressed, 0);
return new String(uncompressed, encoding); return new String(uncompressed, encoding);
} }
@ -797,10 +862,12 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static String uncompressString(byte[] input, int offset, int length, Charset encoding) throws IOException, public static String uncompressString(byte[] input, int offset, int length, Charset encoding)
UnsupportedEncodingException { throws IOException,
UnsupportedEncodingException
{
byte[] uncompressed = new byte[uncompressedLength(input, offset, length)]; byte[] uncompressed = new byte[uncompressedLength(input, offset, length)];
int compressedSize = uncompress(input, offset, length, uncompressed, 0); uncompress(input, offset, length, uncompressed, 0);
return new String(uncompressed, encoding); return new String(uncompressed, encoding);
} }
@ -813,8 +880,10 @@ public class Snappy
* @throws IOException * @throws IOException
* @throws UnsupportedEncodingException * @throws UnsupportedEncodingException
*/ */
public static String uncompressString(byte[] input, String encoding) throws IOException, public static String uncompressString(byte[] input, String encoding)
UnsupportedEncodingException { throws IOException,
UnsupportedEncodingException
{
byte[] uncompressed = uncompress(input); byte[] uncompressed = uncompress(input);
return new String(uncompressed, encoding); return new String(uncompressed, encoding);
} }
@ -827,8 +896,10 @@ public class Snappy
* @return the uncompressed data * @return the uncompressed data
* @throws IOException * @throws IOException
*/ */
public static String uncompressString(byte[] input, Charset encoding) throws IOException, public static String uncompressString(byte[] input, Charset encoding)
UnsupportedEncodingException { throws IOException,
UnsupportedEncodingException
{
byte[] uncompressed = uncompress(input); byte[] uncompressed = uncompress(input);
return new String(uncompressed, encoding); return new String(uncompressed, encoding);
} }

35
src/main/java/org/xerial/snappy/SnappyBundleActivator.java
View File

@ -34,34 +34,35 @@ import java.util.jar.Manifest;
* OSGi bundle entry point * OSGi bundle entry point
* *
* @author leo * @author leo
*
*/ */
public class SnappyBundleActivator implements BundleActivator public class SnappyBundleActivator
implements BundleActivator
{ {
/** /**
* Name of the Snappy native library * Name of the Snappy native library
*/ */
public static final String LIBRARY_NAME = "snappyjava"; public static final String LIBRARY_NAME = "snappyjava";
/** /**
* Make a call to {@link System#loadLibrary(String)} to load the native library which assumes * Make a call to {@link System#loadLibrary(String)} to load the native library which assumes
* that the library is available on the path based on this {@link Bundle}'s {@link Manifest}. * that the library is available on the path based on this {@link Bundle}'s {@link Manifest}.
*/ */
public void start(BundleContext context) throws Exception public void start(BundleContext context)
throws Exception
{ {
String library = System.mapLibraryName(LIBRARY_NAME); String library = System.mapLibraryName(LIBRARY_NAME);
if (library.toLowerCase().endsWith(".dylib")) if (library.toLowerCase().endsWith(".dylib")) {
{ // some MacOS JDK7+ vendors map to dylib instead of jnilib
// some MacOS JDK7+ vendors map to dylib instead of jnilib library = library.replace(".dylib", ".jnilib");
library = library.replace(".dylib", ".jnilib"); }
} System.loadLibrary(library);
System.loadLibrary(library); SnappyLoader.setApi(new SnappyNative());
SnappyLoader.setApi(new SnappyNative());
} }
public void stop(BundleContext context) throws Exception public void stop(BundleContext context)
throws Exception
{ {
SnappyLoader.setApi(null); SnappyLoader.setApi(null);
SnappyLoader.cleanUpExtractedNativeLib(); SnappyLoader.cleanUpExtractedNativeLib();
} }
} }

64
src/main/java/org/xerial/snappy/SnappyCodec.java
View File

@ -34,41 +34,40 @@ import java.util.Arrays;
/** /**
* Preamble header for {@link SnappyOutputStream}. * Preamble header for {@link SnappyOutputStream}.
* * <p/>
* <p> * <p>
* The magic header is the following 8 bytes data: * The magic header is the following 8 bytes data:
* * <p/>
* <pre> * <pre>
* -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 * -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0
* </pre> * </pre>
* * <p/>
* </p> * </p>
* *
* @author leo * @author leo
*
*/ */
public class SnappyCodec public class SnappyCodec
{ {
public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 }; static final byte[] MAGIC_HEADER = new byte[] {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0};
public static final int MAGIC_LEN = MAGIC_HEADER.length; public static final int MAGIC_LEN = MAGIC_HEADER.length;
public static final int HEADER_SIZE = MAGIC_LEN + 8; public static final int HEADER_SIZE = MAGIC_LEN + 8;
public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0); public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0);
public static final int MAGIC_HEADER_TAIL = SnappyOutputStream.readInt(MAGIC_HEADER, 4);
static { static {
assert(MAGIC_HEADER_HEAD < 0); assert (MAGIC_HEADER_HEAD < 0);
} }
public static final int DEFAULT_VERSION = 1;
public static final int MINIMUM_COMPATIBLE_VERSION = 1;
public static final SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
public static final int DEFAULT_VERSION = 1; public final byte[] magic;
public static final int MINIMUM_COMPATIBLE_VERSION = 1; public final int version;
public final int compatibleVersion;
public final byte[] magic;
public final int version;
public final int compatibleVersion;
private final byte[] headerArray; private final byte[] headerArray;
private SnappyCodec(byte[] magic, int version, int compatibleVersion) { private SnappyCodec(byte[] magic, int version, int compatibleVersion)
{
this.magic = magic; this.magic = magic;
this.version = version; this.version = version;
this.compatibleVersion = compatibleVersion; this.compatibleVersion = compatibleVersion;
@ -81,36 +80,49 @@ public class SnappyCodec
d.writeInt(compatibleVersion); d.writeInt(compatibleVersion);
d.close(); d.close();
} }
catch(IOException e) { catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
headerArray = header.toByteArray(); headerArray = header.toByteArray();
} }
public static byte[] getMagicHeader()
{
return MAGIC_HEADER.clone();
}
@Override @Override
public String toString() { public String toString()
{
return String.format("version:%d, compatible version:%d", version, compatibleVersion); return String.format("version:%d, compatible version:%d", version, compatibleVersion);
} }
public static int headerSize() { public static int headerSize()
{
return HEADER_SIZE; return HEADER_SIZE;
} }
public int writeHeader(byte[] dst, int dstOffset) { public int writeHeader(byte[] dst, int dstOffset)
{
System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length); System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length);
return headerArray.length; return headerArray.length;
} }
public int writeHeader(OutputStream out) throws IOException { public int writeHeader(OutputStream out)
throws IOException
{
out.write(headerArray, 0, headerArray.length); out.write(headerArray, 0, headerArray.length);
return headerArray.length; return headerArray.length;
} }
public boolean isValidMagicHeader() { public boolean isValidMagicHeader()
{
return Arrays.equals(MAGIC_HEADER, magic); return Arrays.equals(MAGIC_HEADER, magic);
} }
public static SnappyCodec readHeader(InputStream in) throws IOException { public static SnappyCodec readHeader(InputStream in)
throws IOException
{
DataInputStream d = new DataInputStream(in); DataInputStream d = new DataInputStream(in);
byte[] magic = new byte[MAGIC_LEN]; byte[] magic = new byte[MAGIC_LEN];
d.readFully(magic, 0, MAGIC_LEN); d.readFully(magic, 0, MAGIC_LEN);
@ -118,7 +130,5 @@ public class SnappyCodec
int compatibleVersion = d.readInt(); int compatibleVersion = d.readInt();
return new SnappyCodec(magic, version, compatibleVersion); return new SnappyCodec(magic, version, compatibleVersion);
} }
public static SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
} }

19
src/main/java/org/xerial/snappy/SnappyError.java
View File

@ -28,35 +28,38 @@ package org.xerial.snappy;
* Used when serious errors (unchecked exception) are observed. * Used when serious errors (unchecked exception) are observed.
* *
* @author leo * @author leo
*
*/ */
public class SnappyError extends Error public class SnappyError
extends Error
{ {
/** /**
* *
*/ */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode; public final SnappyErrorCode errorCode;
public SnappyError(SnappyErrorCode code) { public SnappyError(SnappyErrorCode code)
{
super(); super();
this.errorCode = code; this.errorCode = code;
} }
public SnappyError(SnappyErrorCode code, Error e) { public SnappyError(SnappyErrorCode code, Error e)
{
super(e); super(e);
this.errorCode = code; this.errorCode = code;
} }
public SnappyError(SnappyErrorCode code, String message) { public SnappyError(SnappyErrorCode code, String message)
{
super(message); super(message);
this.errorCode = code; this.errorCode = code;
} }
@Override @Override
public String getMessage() { public String getMessage()
{
return String.format("[%s] %s", errorCode.name(), super.getMessage()); return String.format("[%s] %s", errorCode.name(), super.getMessage());
} }
} }

19
src/main/java/org/xerial/snappy/SnappyErrorCode.java
View File

@ -28,9 +28,9 @@ package org.xerial.snappy;
* Error codes of snappy-java * Error codes of snappy-java
* *
* @author leo * @author leo
*
*/ */
public enum SnappyErrorCode { public enum SnappyErrorCode
{
// DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp // DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp
UNKNOWN(0), UNKNOWN(0),
@ -41,24 +41,27 @@ public enum SnappyErrorCode {
FAILED_TO_UNCOMPRESS(5), FAILED_TO_UNCOMPRESS(5),
EMPTY_INPUT(6), EMPTY_INPUT(6),
INCOMPATIBLE_VERSION(7), INCOMPATIBLE_VERSION(7),
INVALID_CHUNK_SIZE(8) INVALID_CHUNK_SIZE(8);
;
public final int id; public final int id;
private SnappyErrorCode(int id) { private SnappyErrorCode(int id)
{
this.id = id; this.id = id;
} }
public static SnappyErrorCode getErrorCode(int id) { public static SnappyErrorCode getErrorCode(int id)
{
for (SnappyErrorCode code : SnappyErrorCode.values()) { for (SnappyErrorCode code : SnappyErrorCode.values()) {
if (code.id == id) if (code.id == id) {
return code; return code;
}
} }
return UNKNOWN; return UNKNOWN;
} }
public static String getErrorMessage(int id) { public static String getErrorMessage(int id)
{
return getErrorCode(id).name(); return getErrorCode(id).name();
} }
} }

30
src/main/java/org/xerial/snappy/SnappyException.java
View File

@ -29,46 +29,54 @@ import java.io.IOException;
/** /**
* Exception in snappy-java * Exception in snappy-java
* *
* @deprecated Snappy-java now uses {@link IOException}
* @author leo * @author leo
* * @deprecated Snappy-java now uses {@link IOException}
*/ */
@Deprecated @Deprecated
public class SnappyException extends Exception public class SnappyException
extends Exception
{ {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode; public final SnappyErrorCode errorCode;
public SnappyException(int code) { public SnappyException(int code)
{
this(SnappyErrorCode.getErrorCode(code)); this(SnappyErrorCode.getErrorCode(code));
} }
public SnappyException(SnappyErrorCode errorCode) { public SnappyException(SnappyErrorCode errorCode)
{
super(); super();
this.errorCode = errorCode; this.errorCode = errorCode;
} }
public SnappyException(SnappyErrorCode errorCode, Exception e) { public SnappyException(SnappyErrorCode errorCode, Exception e)
{
super(e); super(e);
this.errorCode = errorCode; this.errorCode = errorCode;
} }
public SnappyException(SnappyErrorCode errorCode, String message) { public SnappyException(SnappyErrorCode errorCode, String message)
{
super(message); super(message);
this.errorCode = errorCode; this.errorCode = errorCode;
} }
public SnappyErrorCode getErrorCode() { public SnappyErrorCode getErrorCode()
{
return errorCode; return errorCode;
} }
public static void throwException(int errorCode) throws SnappyException { public static void throwException(int errorCode)
throws SnappyException
{
throw new SnappyException(errorCode); throw new SnappyException(errorCode);
} }
@Override @Override
public String getMessage() { public String getMessage()
{
return String.format("[%s] %s", errorCode.name(), super.getMessage()); return String.format("[%s] %s", errorCode.name(), super.getMessage());
} }
} }

51
src/main/java/org/xerial/snappy/SnappyFramed.java
View File

@ -16,7 +16,8 @@ import java.util.logging.Logger;
* @author Brett Okken * @author Brett Okken
* @since 1.1.0 * @since 1.1.0
*/ */
final class SnappyFramed { final class SnappyFramed
{
public static final int COMPRESSED_DATA_FLAG = 0x00; public static final int COMPRESSED_DATA_FLAG = 0x00;
public static final int UNCOMPRESSED_DATA_FLAG = 0x01; public static final int UNCOMPRESSED_DATA_FLAG = 0x01;
@ -33,19 +34,19 @@ final class SnappyFramed {
private static final Method SUN_BUFFER_CLEANER; private static final Method SUN_BUFFER_CLEANER;
private static final Method SUN_CLEANER_CLEAN; private static final Method SUN_CLEANER_CLEAN;
static static {
{
Method bufferCleaner = null; Method bufferCleaner = null;
Method cleanerClean = null; Method cleanerClean = null;
try { try {
//operate under the assumption that if the sun direct buffer class exists, //operate under the assumption that if the sun direct buffer class exists,
//all of the sun classes exist //all of the sun classes exist
if (SUN_DIRECT_BUFFER != null) { if (SUN_DIRECT_BUFFER != null) {
bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[])null); bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[]) null);
Class<?> cleanClazz = lookupClassQuietly("sun.misc.Cleaner"); Class<?> cleanClazz = lookupClassQuietly("sun.misc.Cleaner");
cleanerClean = cleanClazz.getMethod("clean", (Class[])null); cleanerClean = cleanClazz.getMethod("clean", (Class[]) null);
} }
} catch(Throwable t) { }
catch (Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t); Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t);
} }
SUN_BUFFER_CLEANER = bufferCleaner; SUN_BUFFER_CLEANER = bufferCleaner;
@ -58,7 +59,7 @@ final class SnappyFramed {
*/ */
public static final byte[] HEADER_BYTES = new byte[] { public static final byte[] HEADER_BYTES = new byte[] {
(byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
0x50, 0x70, 0x59 }; 0x50, 0x70, 0x59};
public static int maskedCrc32c(byte[] data) public static int maskedCrc32c(byte[] data)
{ {
@ -78,7 +79,7 @@ final class SnappyFramed {
* in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
* 0xa282ead8 (using wraparound as normal for unsigned integers). This is * 0xa282ead8 (using wraparound as normal for unsigned integers). This is
* equivalent to the following C code: * equivalent to the following C code:
* * <p/>
* <pre> * <pre>
* uint32_t mask_checksum(uint32_t x) { * uint32_t mask_checksum(uint32_t x) {
* return ((x >> 15) | (x << 17)) + 0xa282ead8; * return ((x >> 15) | (x << 17)) + 0xa282ead8;
@ -91,8 +92,8 @@ final class SnappyFramed {
return ((crc >>> 15) | (crc << 17)) + MASK_DELTA; return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
} }
static final int readBytes(ReadableByteChannel source, ByteBuffer dest)
static final int readBytes(ReadableByteChannel source, ByteBuffer dest) throws IOException throws IOException
{ {
// tells how many bytes to read. // tells how many bytes to read.
final int expectedLength = dest.remaining(); final int expectedLength = dest.remaining();
@ -105,34 +106,30 @@ final class SnappyFramed {
totalRead = lastRead; totalRead = lastRead;
// if we did not read as many bytes as we had hoped, try reading again. // if we did not read as many bytes as we had hoped, try reading again.
if (lastRead < expectedLength) if (lastRead < expectedLength) {
{
// as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading. // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
while (dest.remaining() != 0 && lastRead != -1) while (dest.remaining() != 0 && lastRead != -1) {
{
lastRead = source.read(dest); lastRead = source.read(dest);
// if we got EOF, do not add to total read. // if we got EOF, do not add to total read.
if (lastRead != -1) if (lastRead != -1) {
{
totalRead += lastRead; totalRead += lastRead;
} }
} }
} }
if (totalRead > 0) if (totalRead > 0) {
{
dest.limit(dest.position()); dest.limit(dest.position());
} }
else else {
{
dest.position(dest.limit()); dest.position(dest.limit());
} }
return totalRead; return totalRead;
} }
static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) throws IOException static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer)
throws IOException
{ {
if (skip <= 0) { if (skip <= 0) {
return 0; return 0;
@ -140,7 +137,7 @@ final class SnappyFramed {
int toSkip = skip; int toSkip = skip;
int skipped = 0; int skipped = 0;
while(toSkip > 0 && skipped != -1) { while (toSkip > 0 && skipped != -1) {
buffer.clear(); buffer.clear();
if (toSkip < buffer.capacity()) { if (toSkip < buffer.capacity()) {
buffer.limit(toSkip); buffer.limit(toSkip);
@ -156,10 +153,12 @@ final class SnappyFramed {
return skip - toSkip; return skip - toSkip;
} }
private static Class<?> lookupClassQuietly(String name) { private static Class<?> lookupClassQuietly(String name)
{
try { try {
return SnappyFramed.class.getClassLoader().loadClass(name); return SnappyFramed.class.getClassLoader().loadClass(name);
} catch (Throwable t) { }
catch (Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t); Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t);
} }
@ -168,6 +167,7 @@ final class SnappyFramed {
/** /**
* Provides jvm implementation specific operation to aggressively release resources associated with <i>buffer</i>. * Provides jvm implementation specific operation to aggressively release resources associated with <i>buffer</i>.
*
* @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}. * @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}.
*/ */
static void releaseDirectByteBuffer(ByteBuffer buffer) static void releaseDirectByteBuffer(ByteBuffer buffer)
@ -178,7 +178,8 @@ final class SnappyFramed {
try { try {
Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null); Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null);
SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null); SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null);
} catch (Throwable t) { }
catch (Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t); Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t);
} }
} }

122
src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
View File

@ -32,8 +32,11 @@ import java.util.Arrays;
* @author Brett Okken * @author Brett Okken
* @since 1.1.0 * @since 1.1.0
*/ */
public final class SnappyFramedInputStream extends InputStream implements public final class SnappyFramedInputStream
ReadableByteChannel { extends InputStream
implements
ReadableByteChannel
{
private final ReadableByteChannel rbc; private final ReadableByteChannel rbc;
private final ByteBuffer frameHeader; private final ByteBuffer frameHeader;
@ -78,10 +81,11 @@ public final class SnappyFramedInputStream extends InputStream implements
* Creates a Snappy input stream to read data from the specified underlying * Creates a Snappy input stream to read data from the specified underlying
* input stream. * input stream.
* *
* @param in * @param in the underlying input stream. Must not be {@code null}.
* the underlying input stream. Must not be {@code null}.
*/ */
public SnappyFramedInputStream(InputStream in) throws IOException { public SnappyFramedInputStream(InputStream in)
throws IOException
{
this(in, true); this(in, true);
} }
@ -89,13 +93,12 @@ public final class SnappyFramedInputStream extends InputStream implements
* Creates a Snappy input stream to read data from the specified underlying * Creates a Snappy input stream to read data from the specified underlying
* input stream. * input stream.
* *
* @param in * @param in the underlying input stream. Must not be {@code null}.
* the underlying input stream. Must not be {@code null}. * @param verifyChecksums if true, checksums in input stream will be verified
* @param verifyChecksums
* if true, checksums in input stream will be verified
*/ */
public SnappyFramedInputStream(InputStream in, boolean verifyChecksums) public SnappyFramedInputStream(InputStream in, boolean verifyChecksums)
throws IOException { throws IOException
{
this(Channels.newChannel(in), verifyChecksums); this(Channels.newChannel(in), verifyChecksums);
} }
@ -103,11 +106,11 @@ public final class SnappyFramedInputStream extends InputStream implements
* Creates a Snappy input stream to read data from the specified underlying * Creates a Snappy input stream to read data from the specified underlying
* channel. * channel.
* *
* @param in * @param in the underlying readable channel. Must not be {@code null}.
* the underlying readable channel. Must not be {@code null}.
*/ */
public SnappyFramedInputStream(ReadableByteChannel in) public SnappyFramedInputStream(ReadableByteChannel in)
throws IOException { throws IOException
{
this(in, true); this(in, true);
} }
@ -115,13 +118,13 @@ public final class SnappyFramedInputStream extends InputStream implements
* Creates a Snappy input stream to read data from the specified underlying * Creates a Snappy input stream to read data from the specified underlying
* channel. * channel.
* *
* @param in * @param in the underlying readable channel. Must not be {@code null}.
* the underlying readable channel. Must not be {@code null}. * @param verifyChecksums if true, checksums in input stream will be verified
* @param verifyChecksums
* if true, checksums in input stream will be verified
*/ */
public SnappyFramedInputStream(ReadableByteChannel in, public SnappyFramedInputStream(ReadableByteChannel in,
boolean verifyChecksums) throws IOException { boolean verifyChecksums)
throws IOException
{
if (in == null) { if (in == null) {
throw new NullPointerException("in is null"); throw new NullPointerException("in is null");
} }
@ -150,7 +153,8 @@ public final class SnappyFramedInputStream extends InputStream implements
/** /**
* @param size * @param size
*/ */
private void allocateBuffersBasedOnSize(int size) { private void allocateBuffersBasedOnSize(int size)
{
if (input != null) { if (input != null) {
releaseDirectByteBuffer(input); releaseDirectByteBuffer(input);
@ -167,7 +171,9 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
@Override @Override
public int read() throws IOException { public int read()
throws IOException
{
if (closed) { if (closed) {
return -1; return -1;
} }
@ -178,7 +184,9 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
@Override @Override
public int read(byte[] output, int offset, int length) throws IOException { public int read(byte[] output, int offset, int length)
throws IOException
{
if (output == null) { if (output == null) {
throw new IllegalArgumentException("output is null"); throw new IllegalArgumentException("output is null");
@ -207,7 +215,9 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
@Override @Override
public int available() throws IOException { public int available()
throws IOException
{
if (closed) { if (closed) {
return 0; return 0;
} }
@ -218,7 +228,8 @@ public final class SnappyFramedInputStream extends InputStream implements
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public boolean isOpen() { public boolean isOpen()
{
return !closed; return !closed;
} }
@ -226,7 +237,9 @@ public final class SnappyFramedInputStream extends InputStream implements
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public int read(ByteBuffer dst) throws IOException { public int read(ByteBuffer dst)
throws IOException
{
if (dst == null) { if (dst == null) {
throw new IllegalArgumentException("dst is null"); throw new IllegalArgumentException("dst is null");
@ -260,13 +273,14 @@ public final class SnappyFramedInputStream extends InputStream implements
* value of {@code 0}. * value of {@code 0}.
* </p> * </p>
* *
* @param os * @param os The destination to write decompressed content to.
* The destination to write decompressed content to.
* @return The number of bytes transferred. * @return The number of bytes transferred.
* @throws IOException * @throws IOException
* @since 1.1.1 * @since 1.1.1
*/ */
public long transferTo(OutputStream os) throws IOException { public long transferTo(OutputStream os)
throws IOException
{
if (os == null) { if (os == null) {
throw new IllegalArgumentException("os is null"); throw new IllegalArgumentException("os is null");
} }
@ -291,7 +305,7 @@ public final class SnappyFramedInputStream extends InputStream implements
* Transfers the entire content of this {@link ReadableByteChannel} to * Transfers the entire content of this {@link ReadableByteChannel} to
* <i>wbc</i>. This potentially limits the amount of buffering required to * <i>wbc</i>. This potentially limits the amount of buffering required to
* decompress content. * decompress content.
* * <p/>
* <p> * <p>
* Unlike {@link #read(ByteBuffer)}, this method does not need to be called * Unlike {@link #read(ByteBuffer)}, this method does not need to be called
* multiple times. A single call will transfer all available content. Any * multiple times. A single call will transfer all available content. Any
@ -299,13 +313,14 @@ public final class SnappyFramedInputStream extends InputStream implements
* of {@code 0}. * of {@code 0}.
* </p> * </p>
* *
* @param wbc * @param wbc The destination to write decompressed content to.
* The destination to write decompressed content to.
* @return The number of bytes transferred. * @return The number of bytes transferred.
* @throws IOException * @throws IOException
* @since 1.1.1 * @since 1.1.1
*/ */
public long transferTo(WritableByteChannel wbc) throws IOException { public long transferTo(WritableByteChannel wbc)
throws IOException
{
if (wbc == null) { if (wbc == null) {
throw new IllegalArgumentException("wbc is null"); throw new IllegalArgumentException("wbc is null");
} }
@ -335,10 +350,13 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
@Override @Override
public void close() throws IOException { public void close()
throws IOException
{
try { try {
rbc.close(); rbc.close();
} finally { }
finally {
if (!closed) { if (!closed) {
closed = true; closed = true;
} }
@ -353,11 +371,13 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
} }
static enum FrameAction { static enum FrameAction
{
RAW, SKIP, UNCOMPRESS; RAW, SKIP, UNCOMPRESS;
} }
public static final class FrameMetaData { public static final class FrameMetaData
{
final int length; final int length;
final FrameAction frameAction; final FrameAction frameAction;
@ -365,14 +385,16 @@ public final class SnappyFramedInputStream extends InputStream implements
* @param frameAction * @param frameAction
* @param length * @param length
*/ */
public FrameMetaData(FrameAction frameAction, int length) { public FrameMetaData(FrameAction frameAction, int length)
{
super(); super();
this.frameAction = frameAction; this.frameAction = frameAction;
this.length = length; this.length = length;
} }
} }
public static final class FrameData { public static final class FrameData
{
final int checkSum; final int checkSum;
final int offset; final int offset;
@ -380,14 +402,17 @@ public final class SnappyFramedInputStream extends InputStream implements
* @param checkSum * @param checkSum
* @param offset * @param offset
*/ */
public FrameData(int checkSum, int offset) { public FrameData(int checkSum, int offset)
{
super(); super();
this.checkSum = checkSum; this.checkSum = checkSum;
this.offset = offset; this.offset = offset;
} }
} }
private boolean ensureBuffer() throws IOException { private boolean ensureBuffer()
throws IOException
{
if (available() > 0) { if (available() > 0) {
return true; return true;
} }
@ -442,7 +467,8 @@ public final class SnappyFramedInputStream extends InputStream implements
uncompressedDirect.get(buffer, 0, valid); uncompressedDirect.get(buffer, 0, valid);
this.position = 0; this.position = 0;
} else { }
else {
// we need to start reading at the offset // we need to start reading at the offset
input.position(frameData.offset); input.position(frameData.offset);
this.position = 0; this.position = 0;
@ -461,7 +487,9 @@ public final class SnappyFramedInputStream extends InputStream implements
return true; return true;
} }
private boolean readBlockHeader() throws IOException { private boolean readBlockHeader()
throws IOException
{
frameHeader.clear(); frameHeader.clear();
int read = readBytes(rbc, frameHeader); int read = readBytes(rbc, frameHeader);
@ -478,13 +506,13 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
/** /**
*
* @param frameHeader * @param frameHeader
* @return * @return
* @throws IOException * @throws IOException
*/ */
private FrameMetaData getFrameMetaData(ByteBuffer frameHeader) private FrameMetaData getFrameMetaData(ByteBuffer frameHeader)
throws IOException { throws IOException
{
assert frameHeader.hasArray(); assert frameHeader.hasArray();
@ -537,16 +565,18 @@ public final class SnappyFramedInputStream extends InputStream implements
} }
/** /**
*
* @param content * @param content
* @return * @return
* @throws IOException * @throws IOException
*/ */
private FrameData getFrameData(ByteBuffer content) throws IOException { private FrameData getFrameData(ByteBuffer content)
throws IOException
{
return new FrameData(getCrc32c(content), 4); return new FrameData(getCrc32c(content), 4);
} }
private int getCrc32c(ByteBuffer content) { private int getCrc32c(ByteBuffer content)
{
final int position = content.position(); final int position = content.position();

154
src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java
View File

@ -28,8 +28,11 @@ import java.nio.channels.WritableByteChannel;
* @author Brett Okken * @author Brett Okken
* @since 1.1.0 * @since 1.1.0
*/ */
public final class SnappyFramedOutputStream extends OutputStream implements public final class SnappyFramedOutputStream
WritableByteChannel { extends OutputStream
implements
WritableByteChannel
{
/** /**
* The x-snappy-framed specification allows for a chunk size up to * The x-snappy-framed specification allows for a chunk size up to
@ -69,32 +72,33 @@ public final class SnappyFramedOutputStream extends OutputStream implements
/** /**
* Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE} * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE}
* and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
* @param out *
* The underlying {@link OutputStream} to write to. Must not be * @param out The underlying {@link OutputStream} to write to. Must not be
* {@code null}. * {@code null}.
* @throws IOException * @throws IOException
*/ */
public SnappyFramedOutputStream(OutputStream out) throws IOException { public SnappyFramedOutputStream(OutputStream out)
throws IOException
{
this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO); this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO);
} }
/** /**
* Creates a new {@link SnappyFramedOutputStream} instance. * Creates a new {@link SnappyFramedOutputStream} instance.
* *
* @param out * @param out The underlying {@link OutputStream} to write to. Must not be
* The underlying {@link OutputStream} to write to. Must not be * {@code null}.
* {@code null}. * @param blockSize The block size (of raw data) to compress before writing frames
* @param blockSize * to <i>out</i>. Must be in (0, 65536].
* The block size (of raw data) to compress before writing frames * @param minCompressionRatio Defines the minimum compression ratio (
* to <i>out</i>. Must be in (0, 65536]. * {@code compressedLength / rawLength}) that must be achieved to
* @param minCompressionRatio * write the compressed data. This must be in (0, 1.0].
* Defines the minimum compression ratio (
* {@code compressedLength / rawLength}) that must be achieved to
* write the compressed data. This must be in (0, 1.0].
* @throws IOException * @throws IOException
*/ */
public SnappyFramedOutputStream(OutputStream out, int blockSize, public SnappyFramedOutputStream(OutputStream out, int blockSize,
double minCompressionRatio) throws IOException { double minCompressionRatio)
throws IOException
{
this(Channels.newChannel(out), blockSize, minCompressionRatio); this(Channels.newChannel(out), blockSize, minCompressionRatio);
} }
@ -102,34 +106,34 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* Creates a new {@link SnappyFramedOutputStream} using the * Creates a new {@link SnappyFramedOutputStream} using the
* {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}. * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
* *
* @param out * @param out The underlying {@link WritableByteChannel} to write to. Must
* The underlying {@link WritableByteChannel} to write to. Must * not be {@code null}.
* not be {@code null}.
* @since 1.1.1
* @throws IOException * @throws IOException
* @since 1.1.1
*/ */
public SnappyFramedOutputStream(WritableByteChannel out) throws IOException { public SnappyFramedOutputStream(WritableByteChannel out)
throws IOException
{
this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO); this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO);
} }
/** /**
* Creates a new {@link SnappyFramedOutputStream} instance. * Creates a new {@link SnappyFramedOutputStream} instance.
* *
* @param out * @param out The underlying {@link WritableByteChannel} to write to. Must
* The underlying {@link WritableByteChannel} to write to. Must * not be {@code null}.
* not be {@code null}. * @param blockSize The block size (of raw data) to compress before writing frames
* @param blockSize * to <i>out</i>. Must be in (0, 65536].
* The block size (of raw data) to compress before writing frames * @param minCompressionRatio Defines the minimum compression ratio (
* to <i>out</i>. Must be in (0, 65536]. * {@code compressedLength / rawLength}) that must be achieved to
* @param minCompressionRatio * write the compressed data. This must be in (0, 1.0].
* Defines the minimum compression ratio (
* {@code compressedLength / rawLength}) that must be achieved to
* write the compressed data. This must be in (0, 1.0].
* @since 1.1.1
* @throws IOException * @throws IOException
* @since 1.1.1
*/ */
public SnappyFramedOutputStream(WritableByteChannel out, int blockSize, public SnappyFramedOutputStream(WritableByteChannel out, int blockSize,
double minCompressionRatio) throws IOException { double minCompressionRatio)
throws IOException
{
if (out == null) { if (out == null) {
throw new NullPointerException(); throw new NullPointerException();
} }
@ -158,11 +162,12 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* Writes the implementation specific header or "marker bytes" to * Writes the implementation specific header or "marker bytes" to
* <i>out</i>. * <i>out</i>.
* *
* @param out * @param out The underlying {@link OutputStream}.
* The underlying {@link OutputStream}.
* @throws IOException * @throws IOException
*/ */
private void writeHeader(WritableByteChannel out) throws IOException { private void writeHeader(WritableByteChannel out)
throws IOException
{
out.write(ByteBuffer.wrap(HEADER_BYTES)); out.write(ByteBuffer.wrap(HEADER_BYTES));
} }
@ -170,12 +175,15 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public boolean isOpen() { public boolean isOpen()
{
return !closed; return !closed;
} }
@Override @Override
public void write(int b) throws IOException { public void write(int b)
throws IOException
{
if (closed) { if (closed) {
throw new IOException("Stream is closed"); throw new IOException("Stream is closed");
} }
@ -186,14 +194,17 @@ public final class SnappyFramedOutputStream extends OutputStream implements
} }
@Override @Override
public void write(byte[] input, int offset, int length) throws IOException { public void write(byte[] input, int offset, int length)
throws IOException
{
if (closed) { if (closed) {
throw new IOException("Stream is closed"); throw new IOException("Stream is closed");
} }
if (input == null) { if (input == null) {
throw new NullPointerException(); throw new NullPointerException();
} else if ((offset < 0) || (offset > input.length) || (length < 0) }
else if ((offset < 0) || (offset > input.length) || (length < 0)
|| ((offset + length) > input.length) || ((offset + length) > input.length)
|| ((offset + length) < 0)) { || ((offset + length) < 0)) {
throw new IndexOutOfBoundsException(); throw new IndexOutOfBoundsException();
@ -215,7 +226,9 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public int write(ByteBuffer src) throws IOException { public int write(ByteBuffer src)
throws IOException
{
if (closed) { if (closed) {
throw new ClosedChannelException(); throw new ClosedChannelException();
} }
@ -256,13 +269,14 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* This potentially limits the amount of buffering required to compress * This potentially limits the amount of buffering required to compress
* content. * content.
* *
* @param is * @param is The source of data to compress.
* The source of data to compress.
* @return The number of bytes read from <i>is</i>. * @return The number of bytes read from <i>is</i>.
* @throws IOException * @throws IOException
* @since 1.1.1 * @since 1.1.1
*/ */
public long transferFrom(InputStream is) throws IOException { public long transferFrom(InputStream is)
throws IOException
{
if (closed) { if (closed) {
throw new ClosedChannelException(); throw new ClosedChannelException();
} }
@ -300,13 +314,14 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* {@link WritableByteChannel}. This potentially limits the amount of * {@link WritableByteChannel}. This potentially limits the amount of
* buffering required to compress content. * buffering required to compress content.
* *
* @param rbc * @param rbc The source of data to compress.
* The source of data to compress.
* @return The number of bytes read from <i>rbc</i>. * @return The number of bytes read from <i>rbc</i>.
* @throws IOException * @throws IOException
* @since 1.1.1 * @since 1.1.1
*/ */
public long transferFrom(ReadableByteChannel rbc) throws IOException { public long transferFrom(ReadableByteChannel rbc)
throws IOException
{
if (closed) { if (closed) {
throw new ClosedChannelException(); throw new ClosedChannelException();
} }
@ -333,7 +348,9 @@ public final class SnappyFramedOutputStream extends OutputStream implements
} }
@Override @Override
public final void flush() throws IOException { public final void flush()
throws IOException
{
if (closed) { if (closed) {
throw new IOException("Stream is closed"); throw new IOException("Stream is closed");
} }
@ -341,14 +358,17 @@ public final class SnappyFramedOutputStream extends OutputStream implements
} }
@Override @Override
public final void close() throws IOException { public final void close()
throws IOException
{
if (closed) { if (closed) {
return; return;
} }
try { try {
flush(); flush();
out.close(); out.close();
} finally { }
finally {
closed = true; closed = true;
releaseDirectByteBuffer(directInputBuffer); releaseDirectByteBuffer(directInputBuffer);
@ -362,7 +382,9 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* *
* @throws IOException * @throws IOException
*/ */
private void flushBuffer() throws IOException { private void flushBuffer()
throws IOException
{
if (buffer.position() > 0) { if (buffer.position() > 0) {
buffer.flip(); buffer.flip();
writeCompressed(buffer); writeCompressed(buffer);
@ -377,10 +399,11 @@ public final class SnappyFramedOutputStream extends OutputStream implements
* actually write the frame. * actually write the frame.
* *
* @param buffer * @param buffer
* @throws IOException * @throws IOException
*/ */
private void writeCompressed(ByteBuffer buffer) throws IOException { private void writeCompressed(ByteBuffer buffer)
throws IOException
{
final byte[] input = buffer.array(); final byte[] input = buffer.array();
final int length = buffer.remaining(); final int length = buffer.remaining();
@ -401,7 +424,8 @@ public final class SnappyFramedOutputStream extends OutputStream implements
// minCompressonRatio // minCompressonRatio
if (((double) compressedLength / (double) length) <= minCompressionRatio) { if (((double) compressedLength / (double) length) <= minCompressionRatio) {
writeBlock(out, outputBuffer, true, crc32c); writeBlock(out, outputBuffer, true, crc32c);
} else { }
else {
// otherwise use the uncompressed data. // otherwise use the uncompressed data.
buffer.flip(); buffer.flip();
writeBlock(out, buffer, false, crc32c); writeBlock(out, buffer, false, crc32c);
@ -411,20 +435,18 @@ public final class SnappyFramedOutputStream extends OutputStream implements
/** /**
* Write a frame (block) to <i>out</i>. * Write a frame (block) to <i>out</i>.
* *
* @param out * @param out The {@link OutputStream} to write to.
* The {@link OutputStream} to write to. * @param data The data to write.
* @param data * @param compressed Indicates if <i>data</i> is the compressed or raw content.
* The data to write. * This is based on whether the compression ratio desired is
* @param compressed * reached.
* Indicates if <i>data</i> is the compressed or raw content. * @param crc32c The calculated checksum.
* This is based on whether the compression ratio desired is
* reached.
* @param crc32c
* The calculated checksum.
* @throws IOException * @throws IOException
*/ */
private void writeBlock(final WritableByteChannel out, ByteBuffer data, private void writeBlock(final WritableByteChannel out, ByteBuffer data,
boolean compressed, int crc32c) throws IOException { boolean compressed, int crc32c)
throws IOException
{
headerBuffer.clear(); headerBuffer.clear();
headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG

11
src/main/java/org/xerial/snappy/SnappyIOException.java
View File

@ -5,19 +5,22 @@ import java.io.IOException;
/** /**
* Enhanced IOException with SnappyErrorCode * Enhanced IOException with SnappyErrorCode
*/ */
public class SnappyIOException extends IOException { public class SnappyIOException
extends IOException
{
private final SnappyErrorCode errorCode; private final SnappyErrorCode errorCode;
public SnappyIOException(SnappyErrorCode errorCode, String message) { public SnappyIOException(SnappyErrorCode errorCode, String message)
{
super(message); super(message);
this.errorCode = errorCode; this.errorCode = errorCode;
} }
@Override @Override
public String getMessage() { public String getMessage()
{
return String.format("[%s] %s", errorCode.name(), super.getMessage()); return String.format("[%s] %s", errorCode.name(), super.getMessage());
} }
public SnappyErrorCode getErrorCode() { return errorCode; } public SnappyErrorCode getErrorCode() { return errorCode; }
} }

227
src/main/java/org/xerial/snappy/SnappyInputStream.java
View File

@ -31,21 +31,20 @@ import java.io.InputStream;
/** /**
* A stream filter for reading data compressed by {@link SnappyOutputStream}. * A stream filter for reading data compressed by {@link SnappyOutputStream}.
* *
*
* @author leo * @author leo
*
*/ */
public class SnappyInputStream extends InputStream public class SnappyInputStream
extends InputStream
{ {
private boolean finishedReading = false; private boolean finishedReading = false;
protected final InputStream in; protected final InputStream in;
private byte[] compressed; private byte[] compressed;
private byte[] uncompressed; private byte[] uncompressed;
private int uncompressedCursor = 0; private int uncompressedCursor = 0;
private int uncompressedLimit = 0; private int uncompressedLimit = 0;
private byte[] header = new byte[SnappyCodec.headerSize()]; private byte[] header = new byte[SnappyCodec.headerSize()];
/** /**
* Create a filter for reading compressed data as a uncompressed stream * Create a filter for reading compressed data as a uncompressed stream
@ -53,7 +52,9 @@ public class SnappyInputStream extends InputStream
* @param input * @param input
* @throws IOException * @throws IOException
*/ */
public SnappyInputStream(InputStream input) throws IOException { public SnappyInputStream(InputStream input)
throws IOException
{
this.in = input; this.in = input;
readHeader(); readHeader();
} }
@ -65,57 +66,63 @@ public class SnappyInputStream extends InputStream
* @see java.io.InputStream#close() * @see java.io.InputStream#close()
*/ */
@Override @Override
public void close() throws IOException { public void close()
throws IOException
{
compressed = null; compressed = null;
uncompressed = null; uncompressed = null;
if (in != null) if (in != null) {
in.close(); in.close();
}
} }
protected void readHeader() throws IOException { protected void readHeader()
throws IOException
{
int readBytes = 0; int readBytes = 0;
while (readBytes < header.length) { while (readBytes < header.length) {
int ret = in.read(header, readBytes, header.length - readBytes); int ret = in.read(header, readBytes, header.length - readBytes);
if (ret == -1) if (ret == -1) {
break; break;
}
readBytes += ret; readBytes += ret;
} }
// Quick test of the header // Quick test of the header
if(readBytes == 0) { if (readBytes == 0) {
// Snappy produces at least 1-byte result. So the empty input is not a valid input // Snappy produces at least 1-byte result. So the empty input is not a valid input
throw new SnappyIOException(SnappyErrorCode.EMPTY_INPUT, "Cannot decompress empty stream"); throw new SnappyIOException(SnappyErrorCode.EMPTY_INPUT, "Cannot decompress empty stream");
} }
if (readBytes < header.length || header[0] != SnappyCodec.MAGIC_HEADER[0]) { if (readBytes < header.length || header[0] != SnappyCodec.MAGIC_HEADER[0]) {
// do the default uncompression // do the default uncompression
readFully(header, readBytes);
return;
}
if(!isValidHeader(header)) {
// (probably) compressed by Snappy.compress(byte[]) // (probably) compressed by Snappy.compress(byte[])
readFully(header, readBytes); readFully(header, readBytes);
return; return;
} }
} }
private static boolean isValidHeader(byte[] header) throws IOException { private static boolean isValidHeader(byte[] header)
throws IOException
{
SnappyCodec codec = SnappyCodec.readHeader(new ByteArrayInputStream(header)); SnappyCodec codec = SnappyCodec.readHeader(new ByteArrayInputStream(header));
if (codec.isValidMagicHeader()) { if (codec.isValidMagicHeader()) {
// The input data is compressed by SnappyOutputStream // The input data is compressed by SnappyOutputStream
if(codec.version < SnappyCodec.MINIMUM_COMPATIBLE_VERSION) { if (codec.version < SnappyCodec.MINIMUM_COMPATIBLE_VERSION) {
throw new SnappyIOException(SnappyErrorCode.INCOMPATIBLE_VERSION, String.format( throw new SnappyIOException(SnappyErrorCode.INCOMPATIBLE_VERSION, String.format(
"Compressed with an incompatible codec version %d. At least version %d is required", "Compressed with an incompatible codec version %d. At least version %d is required",
codec.version, SnappyCodec.MINIMUM_COMPATIBLE_VERSION)); codec.version, SnappyCodec.MINIMUM_COMPATIBLE_VERSION));
} }
return true; return true;
} }
else else {
return false; return false;
}
} }
protected void readFully(byte[] fragment, int fragmentLength) throws IOException { protected void readFully(byte[] fragment, int fragmentLength)
if(fragmentLength == 0) { throws IOException
{
if (fragmentLength == 0) {
finishedReading = true; finishedReading = true;
return; return;
} }
@ -123,7 +130,7 @@ public class SnappyInputStream extends InputStream
compressed = new byte[Math.max(8 * 1024, fragmentLength)]; // 8K compressed = new byte[Math.max(8 * 1024, fragmentLength)]; // 8K
System.arraycopy(fragment, 0, compressed, 0, fragmentLength); System.arraycopy(fragment, 0, compressed, 0, fragmentLength);
int cursor = fragmentLength; int cursor = fragmentLength;
for (int readBytes = 0; (readBytes = in.read(compressed, cursor, compressed.length - cursor)) != -1;) { for (int readBytes = 0; (readBytes = in.read(compressed, cursor, compressed.length - cursor)) != -1; ) {
cursor += readBytes; cursor += readBytes;
if (cursor >= compressed.length) { if (cursor >= compressed.length) {
byte[] newBuf = new byte[(compressed.length * 2)]; byte[] newBuf = new byte[(compressed.length * 2)];
@ -140,7 +147,6 @@ public class SnappyInputStream extends InputStream
Snappy.uncompress(compressed, 0, cursor, uncompressed, 0); Snappy.uncompress(compressed, 0, cursor, uncompressed, 0);
this.uncompressedCursor = 0; this.uncompressedCursor = 0;
this.uncompressedLimit = uncompressedLength; this.uncompressedLimit = uncompressedLength;
} }
/** /**
@ -151,7 +157,9 @@ public class SnappyInputStream extends InputStream
* @see java.io.InputStream#read(byte[], int, int) * @see java.io.InputStream#read(byte[], int, int)
*/ */
@Override @Override
public int read(byte[] b, int off, int len) throws IOException { public int read(byte[] b, int off, int len)
throws IOException
{
return rawRead(b, off, len); return rawRead(b, off, len);
} }
@ -164,13 +172,16 @@ public class SnappyInputStream extends InputStream
* @return written bytes * @return written bytes
* @throws IOException * @throws IOException
*/ */
public int rawRead(Object array, int byteOffset, int byteLength) throws IOException { public int rawRead(Object array, int byteOffset, int byteLength)
throws IOException
{
int writtenBytes = 0; int writtenBytes = 0;
for (; writtenBytes < byteLength;) { for (; writtenBytes < byteLength; ) {
if (uncompressedCursor >= uncompressedLimit) { if (uncompressedCursor >= uncompressedLimit) {
if (hasNextChunk()) if (hasNextChunk()) {
continue; continue;
}
else { else {
return writtenBytes == 0 ? -1 : writtenBytes; return writtenBytes == 0 ? -1 : writtenBytes;
} }
@ -187,17 +198,16 @@ public class SnappyInputStream extends InputStream
/** /**
* Read long array from the stream * Read long array from the stream
* *
* @param d * @param d input
* input * @param off offset
* @param off * @param len the number of long elements to read
* offset
* @param len
* the number of long elements to read
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(long[] d, int off, int len) throws IOException { public int read(long[] d, int off, int len)
throws IOException
{
return rawRead(d, off * 8, len * 8); return rawRead(d, off * 8, len * 8);
} }
@ -206,27 +216,28 @@ public class SnappyInputStream extends InputStream
* *
* @param d * @param d
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(long[] d) throws IOException { public int read(long[] d)
throws IOException
{
return read(d, 0, d.length); return read(d, 0, d.length);
} }
/** /**
* Read double array from the stream * Read double array from the stream
* *
* @param d * @param d input
* input * @param off offset
* @param off * @param len the number of double elements to read
* offset
* @param len
* the number of double elements to read
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(double[] d, int off, int len) throws IOException { public int read(double[] d, int off, int len)
throws IOException
{
return rawRead(d, off * 8, len * 8); return rawRead(d, off * 8, len * 8);
} }
@ -235,10 +246,12 @@ public class SnappyInputStream extends InputStream
* *
* @param d * @param d
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(double[] d) throws IOException { public int read(double[] d)
throws IOException
{
return read(d, 0, d.length); return read(d, 0, d.length);
} }
@ -247,44 +260,44 @@ public class SnappyInputStream extends InputStream
* *
* @param d * @param d
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(int[] d) throws IOException { public int read(int[] d)
throws IOException
{
return read(d, 0, d.length); return read(d, 0, d.length);
} }
/** /**
* Read int array from the stream * Read int array from the stream
* *
* @param d * @param d input
* input * @param off offset
* @param off * @param len the number of int elements to read
* offset
* @param len
* the number of int elements to read
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(int[] d, int off, int len) throws IOException { public int read(int[] d, int off, int len)
throws IOException
{
return rawRead(d, off * 4, len * 4); return rawRead(d, off * 4, len * 4);
} }
/** /**
* Read float array from the stream * Read float array from the stream
* *
* @param d * @param d input
* input * @param off offset
* @param off * @param len the number of float elements to read
* offset
* @param len
* the number of float elements to read
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(float[] d, int off, int len) throws IOException { public int read(float[] d, int off, int len)
throws IOException
{
return rawRead(d, off * 4, len * 4); return rawRead(d, off * 4, len * 4);
} }
@ -293,27 +306,28 @@ public class SnappyInputStream extends InputStream
* *
* @param d * @param d
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(float[] d) throws IOException { public int read(float[] d)
throws IOException
{
return read(d, 0, d.length); return read(d, 0, d.length);
} }
/** /**
* Read short array from the stream * Read short array from the stream
* *
* @param d * @param d input
* input * @param off offset
* @param off * @param len the number of short elements to read
* offset
* @param len
* the number of short elements to read
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(short[] d, int off, int len) throws IOException { public int read(short[] d, int off, int len)
throws IOException
{
return rawRead(d, off * 2, len * 2); return rawRead(d, off * 2, len * 2);
} }
@ -322,21 +336,26 @@ public class SnappyInputStream extends InputStream
* *
* @param d * @param d
* @return the total number of bytes read into the buffer, or -1 if there is * @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached. * no more data because the end of the stream has been reached.
* @throws IOException * @throws IOException
*/ */
public int read(short[] d) throws IOException { public int read(short[] d)
throws IOException
{
return read(d, 0, d.length); return read(d, 0, d.length);
} }
/** /**
* Read next len bytes * Read next len bytes
*
* @param dest * @param dest
* @param offset * @param offset
* @param len * @param len
* @return read bytes * @return read bytes
*/ */
private int readNext(byte[] dest, int offset, int len) throws IOException { private int readNext(byte[] dest, int offset, int len)
throws IOException
{
int readBytes = 0; int readBytes = 0;
while (readBytes < len) { while (readBytes < len) {
int ret = in.read(dest, readBytes + offset, len - readBytes); int ret = in.read(dest, readBytes + offset, len - readBytes);
@ -349,29 +368,36 @@ public class SnappyInputStream extends InputStream
return readBytes; return readBytes;
} }
protected boolean hasNextChunk() throws IOException { protected boolean hasNextChunk()
if (finishedReading) throws IOException
{
if (finishedReading) {
return false; return false;
}
uncompressedCursor = 0; uncompressedCursor = 0;
uncompressedLimit = 0; uncompressedLimit = 0;
int readBytes = readNext(header, 0, 4); int readBytes = readNext(header, 0, 4);
if(readBytes < 4) if (readBytes < 4) {
return false; return false;
}
int chunkSize = SnappyOutputStream.readInt(header, 0); int chunkSize = SnappyOutputStream.readInt(header, 0);
if(chunkSize == SnappyCodec.MAGIC_HEADER_HEAD) { if (chunkSize == SnappyCodec.MAGIC_HEADER_HEAD) {
// Concatenated data // Concatenated data
int remainingHeaderSize = SnappyCodec.headerSize() - 4; int remainingHeaderSize = SnappyCodec.headerSize() - 4;
readBytes = readNext(header, 4, remainingHeaderSize); readBytes = readNext(header, 4, remainingHeaderSize);
if(readBytes < remainingHeaderSize) if(readBytes < remainingHeaderSize) {
return false; throw new SnappyIOException(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format("Insufficient header size in a concatenated block"));
}
if(isValidHeader(header)) if (isValidHeader(header)) {
return hasNextChunk(); return hasNextChunk();
else }
else {
return false; return false;
}
} }
// extend the compressed data buffer size // extend the compressed data buffer size
@ -381,8 +407,9 @@ public class SnappyInputStream extends InputStream
readBytes = 0; readBytes = 0;
while (readBytes < chunkSize) { while (readBytes < chunkSize) {
int ret = in.read(compressed, readBytes, chunkSize - readBytes); int ret = in.read(compressed, readBytes, chunkSize - readBytes);
if (ret == -1) if (ret == -1) {
break; break;
}
readBytes += ret; readBytes += ret;
} }
if (readBytes < chunkSize) { if (readBytes < chunkSize) {
@ -412,15 +439,19 @@ public class SnappyInputStream extends InputStream
* @see java.io.InputStream#read() * @see java.io.InputStream#read()
*/ */
@Override @Override
public int read() throws IOException { public int read()
throws IOException
{
if (uncompressedCursor < uncompressedLimit) { if (uncompressedCursor < uncompressedLimit) {
return uncompressed[uncompressedCursor++] & 0xFF; return uncompressed[uncompressedCursor++] & 0xFF;
} }
else { else {
if (hasNextChunk()) if (hasNextChunk()) {
return read(); return read();
else }
else {
return -1; return -1;
}
} }
} }
@ -428,7 +459,9 @@ public class SnappyInputStream extends InputStream
* @see java.io.InputStream#available() * @see java.io.InputStream#available()
*/ */
@Override @Override
public int available() throws IOException { public int available()
throws IOException
{
if (uncompressedCursor < uncompressedLimit) { if (uncompressedCursor < uncompressedLimit) {
return uncompressedLimit - uncompressedCursor; return uncompressedLimit - uncompressedCursor;
} }

175
src/main/java/org/xerial/snappy/SnappyLoader.java
View File

@ -36,10 +36,10 @@ import java.util.UUID;
* user platform (<i>os.name</i> and <i>os.arch</i>). The natively compiled * user platform (<i>os.name</i> and <i>os.arch</i>). The natively compiled
* libraries bundled to snappy-java contain the codes of the original snappy and * libraries bundled to snappy-java contain the codes of the original snappy and
* JNI programs to access Snappy. * JNI programs to access Snappy.
* * <p/>
* In default, no configuration is required to use snappy-java, but you can load * In default, no configuration is required to use snappy-java, but you can load
* your own native library created by 'make native' command. * your own native library created by 'make native' command.
* * <p/>
* This SnappyLoader searches for native libraries (snappyjava.dll, * This SnappyLoader searches for native libraries (snappyjava.dll,
* libsnappy.so, etc.) in the following order: * libsnappy.so, etc.) in the following order:
* <ol> * <ol>
@ -53,41 +53,45 @@ import java.util.UUID;
* <i>org.xerial.snappy.tempdir</i> is set, use this folder instead of * <i>org.xerial.snappy.tempdir</i> is set, use this folder instead of
* <i>java.io.tempdir</i>. * <i>java.io.tempdir</i>.
* </ol> * </ol>
* * <p/>
* <p> * <p>
* If you do not want to use folder <i>java.io.tempdir</i>, set the System * If you do not want to use folder <i>java.io.tempdir</i>, set the System
* property <i>org.xerial.snappy.tempdir</i>. For example, to use * property <i>org.xerial.snappy.tempdir</i>. For example, to use
* <i>/tmp/leo</i> as a temporary folder to copy native libraries, use -D option * <i>/tmp/leo</i> as a temporary folder to copy native libraries, use -D option
* of JVM: * of JVM:
* * <p/>
* <pre> * <pre>
* <code> * <code>
* java -Dorg.xerial.snappy.tempdir="/tmp/leo" ... * java -Dorg.xerial.snappy.tempdir="/tmp/leo" ...
* </code> * </code>
* </pre> * </pre>
* * <p/>
* </p> * </p>
* *
* @author leo * @author leo
*
*/ */
public class SnappyLoader public class SnappyLoader
{ {
public static final String SNAPPY_SYSTEM_PROPERTIES_FILE = "org-xerial-snappy.properties"; public static final String SNAPPY_SYSTEM_PROPERTIES_FILE = "org-xerial-snappy.properties";
public static final String KEY_SNAPPY_LIB_PATH = "org.xerial.snappy.lib.path"; public static final String KEY_SNAPPY_LIB_PATH = "org.xerial.snappy.lib.path";
public static final String KEY_SNAPPY_LIB_NAME = "org.xerial.snappy.lib.name"; public static final String KEY_SNAPPY_LIB_NAME = "org.xerial.snappy.lib.name";
public static final String KEY_SNAPPY_TEMPDIR = "org.xerial.snappy.tempdir"; public static final String KEY_SNAPPY_TEMPDIR = "org.xerial.snappy.tempdir";
public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib"; public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib";
public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility
private static volatile boolean isLoaded = false; private static volatile boolean isLoaded = false;
private static volatile SnappyNative api = null; private static volatile SnappyNative api = null;
private static File nativeLibFile = null; private static File nativeLibFile = null;
static void cleanUpExtractedNativeLib() { static void cleanUpExtractedNativeLib()
if(nativeLibFile != null && nativeLibFile.exists()) {
nativeLibFile.delete(); if (nativeLibFile != null && nativeLibFile.exists()) {
boolean deleted = nativeLibFile.delete();
if (!deleted) {
// Deleting native lib has failed, but it's not serious so simply ignore it here
}
}
} }
/** /**
@ -97,26 +101,28 @@ public class SnappyLoader
*/ */
static synchronized void setApi(SnappyNative nativeCode) static synchronized void setApi(SnappyNative nativeCode)
{ {
api = nativeCode; api = nativeCode;
} }
/** /**
* load system properties when configuration file of the name * load system properties when configuration file of the name
* {@link #SNAPPY_SYSTEM_PROPERTIES_FILE} is found * {@link #SNAPPY_SYSTEM_PROPERTIES_FILE} is found
*/ */
private static void loadSnappySystemProperties() { private static void loadSnappySystemProperties()
{
try { try {
InputStream is = Thread.currentThread().getContextClassLoader() InputStream is = Thread.currentThread().getContextClassLoader()
.getResourceAsStream(SNAPPY_SYSTEM_PROPERTIES_FILE); .getResourceAsStream(SNAPPY_SYSTEM_PROPERTIES_FILE);
if (is == null) if (is == null) {
return; // no configuration file is found return; // no configuration file is found
}
// Load property file // Load property file
Properties props = new Properties(); Properties props = new Properties();
props.load(is); props.load(is);
is.close(); is.close();
Enumeration< ? > names = props.propertyNames(); Enumeration<?> names = props.propertyNames();
while (names.hasMoreElements()) { while (names.hasMoreElements()) {
String name = (String) names.nextElement(); String name = (String) names.nextElement();
if (name.startsWith("org.xerial.snappy.")) { if (name.startsWith("org.xerial.snappy.")) {
@ -138,8 +144,9 @@ public class SnappyLoader
static synchronized SnappyNative load() static synchronized SnappyNative load()
{ {
if (api != null) if (api != null) {
return api; return api;
}
try { try {
loadNativeLibrary(); loadNativeLibrary();
@ -158,7 +165,8 @@ public class SnappyLoader
/** /**
* Load a native library of snappy-java * Load a native library of snappy-java
*/ */
private static void loadNativeLibrary() { private static void loadNativeLibrary()
{
nativeLibFile = findNativeLibrary(); nativeLibFile = findNativeLibrary();
if (nativeLibFile != null) { if (nativeLibFile != null) {
@ -171,25 +179,28 @@ public class SnappyLoader
} }
} }
private static boolean contentsEquals(InputStream in1, InputStream in2)
private static boolean contentsEquals(InputStream in1, InputStream in2) throws IOException { throws IOException
if(!(in1 instanceof BufferedInputStream)) { {
in1 = new BufferedInputStream(in1); if (!(in1 instanceof BufferedInputStream)) {
in1 = new BufferedInputStream(in1);
} }
if(!(in2 instanceof BufferedInputStream)) { if (!(in2 instanceof BufferedInputStream)) {
in2 = new BufferedInputStream(in2); in2 = new BufferedInputStream(in2);
} }
int ch = in1.read(); int ch = in1.read();
while(ch != -1) { while (ch != -1) {
int ch2 = in2.read(); int ch2 = in2.read();
if(ch != ch2) if (ch != ch2) {
return false; return false;
}
ch = in1.read(); ch = in1.read();
} }
int ch2 = in2.read(); int ch2 = in2.read();
return ch2 == -1; return ch2 == -1;
} }
/** /**
* Extract the specified library file to the target folder * Extract the specified library file to the target folder
* *
@ -198,7 +209,8 @@ public class SnappyLoader
* @param targetFolder * @param targetFolder
* @return * @return
*/ */
private static File extractLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder) { private static File extractLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder)
{
String nativeLibraryFilePath = libFolderForCurrentOS + "/" + libraryFileName; String nativeLibraryFilePath = libFolderForCurrentOS + "/" + libraryFileName;
// Attach UUID to the native library file to ensure multiple class loaders can read the libsnappy-java multiple times. // Attach UUID to the native library file to ensure multiple class loaders can read the libsnappy-java multiple times.
@ -208,44 +220,61 @@ public class SnappyLoader
try { try {
// Extract a native library file into the target directory // Extract a native library file into the target directory
InputStream reader = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath); InputStream reader = null;
FileOutputStream writer = new FileOutputStream(extractedLibFile); FileOutputStream writer = null;
try { try {
byte[] buffer = new byte[8192]; reader = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath);
int bytesRead = 0; try {
while ((bytesRead = reader.read(buffer)) != -1) { writer = new FileOutputStream(extractedLibFile);
writer.write(buffer, 0, bytesRead);
byte[] buffer = new byte[8192];
int bytesRead = 0;
while ((bytesRead = reader.read(buffer)) != -1) {
writer.write(buffer, 0, bytesRead);
}
}
finally {
if (writer != null) {
writer.close();
}
} }
} }
finally { finally {
if (reader != null) {
reader.close();
}
// Delete the extracted lib file on JVM exit. // Delete the extracted lib file on JVM exit.
extractedLibFile.deleteOnExit(); extractedLibFile.deleteOnExit();
if(writer != null)
writer.close();
if(reader != null)
reader.close();
} }
// Set executable (x) flag to enable Java to load the native library // Set executable (x) flag to enable Java to load the native library
extractedLibFile.setReadable(true); boolean success = extractedLibFile.setReadable(true) &&
extractedLibFile.setWritable(true, true); extractedLibFile.setWritable(true, true) &&
extractedLibFile.setExecutable(true); extractedLibFile.setExecutable(true);
if (!success) {
// Setting file flag may fail, but in this case another error will be thrown in later phase
}
// Check whether the contents are properly copied from the resource folder // Check whether the contents are properly copied from the resource folder
{ {
InputStream nativeIn = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath); InputStream nativeIn = null;
InputStream extractedLibIn = new FileInputStream(extractedLibFile); InputStream extractedLibIn = null;
try { try {
if(!contentsEquals(nativeIn, extractedLibIn)) nativeIn = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath);
extractedLibIn = new FileInputStream(extractedLibFile);
if (!contentsEquals(nativeIn, extractedLibIn)) {
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, String.format("Failed to write a native library file at %s", extractedLibFile)); throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, String.format("Failed to write a native library file at %s", extractedLibFile));
}
} }
finally { finally {
if(nativeIn != null) if (nativeIn != null) {
nativeIn.close(); nativeIn.close();
if(extractedLibIn != null) }
if (extractedLibIn != null) {
extractedLibIn.close(); extractedLibIn.close();
}
} }
} }
@ -257,44 +286,47 @@ public class SnappyLoader
} }
} }
static File findNativeLibrary() { static File findNativeLibrary()
{
boolean useSystemLib = Boolean.parseBoolean(System.getProperty(KEY_SNAPPY_USE_SYSTEMLIB, "false")); boolean useSystemLib = Boolean.parseBoolean(System.getProperty(KEY_SNAPPY_USE_SYSTEMLIB, "false"));
boolean disabledBundledLibs = Boolean boolean disabledBundledLibs = Boolean
.parseBoolean(System.getProperty(KEY_SNAPPY_DISABLE_BUNDLED_LIBS, "false")); .parseBoolean(System.getProperty(KEY_SNAPPY_DISABLE_BUNDLED_LIBS, "false"));
if (useSystemLib || disabledBundledLibs) if (useSystemLib || disabledBundledLibs) {
return null; // Use a pre-installed libsnappyjava return null; // Use a pre-installed libsnappyjava
}
// Try to load the library in org.xerial.snappy.lib.path */ // Try to load the library in org.xerial.snappy.lib.path */
String snappyNativeLibraryPath = System.getProperty(KEY_SNAPPY_LIB_PATH); String snappyNativeLibraryPath = System.getProperty(KEY_SNAPPY_LIB_PATH);
String snappyNativeLibraryName = System.getProperty(KEY_SNAPPY_LIB_NAME); String snappyNativeLibraryName = System.getProperty(KEY_SNAPPY_LIB_NAME);
// Resolve the library file name with a suffix (e.g., dll, .so, etc.) // Resolve the library file name with a suffix (e.g., dll, .so, etc.)
if (snappyNativeLibraryName == null) if (snappyNativeLibraryName == null) {
snappyNativeLibraryName = System.mapLibraryName("snappyjava"); snappyNativeLibraryName = System.mapLibraryName("snappyjava");
}
if (snappyNativeLibraryPath != null) { if (snappyNativeLibraryPath != null) {
File nativeLib = new File(snappyNativeLibraryPath, snappyNativeLibraryName); File nativeLib = new File(snappyNativeLibraryPath, snappyNativeLibraryName);
if (nativeLib.exists()) if (nativeLib.exists()) {
return nativeLib; return nativeLib;
}
} }
// Load an OS-dependent native library inside a jar file // Load an OS-dependent native library inside a jar file
snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS(); snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName); boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);
if(!hasNativeLib) { if (!hasNativeLib) {
if(OSInfo.getOSName().equals("Mac")) { if (OSInfo.getOSName().equals("Mac")) {
// Fix for openjdk7 for Mac // Fix for openjdk7 for Mac
String altName = "libsnappyjava.jnilib"; String altName = "libsnappyjava.jnilib";
if(hasResource(snappyNativeLibraryPath + "/" + altName)) { if (hasResource(snappyNativeLibraryPath + "/" + altName)) {
snappyNativeLibraryName = altName; snappyNativeLibraryName = altName;
hasNativeLib = true; hasNativeLib = true;
} }
} }
} }
if(!hasNativeLib) { if (!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(), OSInfo.getArchName()); String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(), OSInfo.getArchName());
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, errorMessage); throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, errorMessage);
} }
@ -302,20 +334,21 @@ public class SnappyLoader
// Temporary folder for the native lib. Use the value of org.xerial.snappy.tempdir or java.io.tmpdir // Temporary folder for the native lib. Use the value of org.xerial.snappy.tempdir or java.io.tmpdir
File tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR, System.getProperty("java.io.tmpdir"))); File tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR, System.getProperty("java.io.tmpdir")));
if (!tempFolder.exists()) { if (!tempFolder.exists()) {
tempFolder.mkdir(); boolean created = tempFolder.mkdirs();
if (!created) {
// if created == false, it will fail eventually in the later part
}
} }
// Extract and load a native library inside the jar file // Extract and load a native library inside the jar file
return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder.getAbsolutePath()); return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder.getAbsolutePath());
} }
private static boolean hasResource(String path)
private static boolean hasResource(String path) { {
return SnappyLoader.class.getResource(path) != null; return SnappyLoader.class.getResource(path) != null;
} }
/** /**
* Get the snappy-java version by reading pom.properties embedded in jar. * Get the snappy-java version by reading pom.properties embedded in jar.
* This version data is used as a suffix of a dll file extracted from the * This version data is used as a suffix of a dll file extracted from the
@ -323,12 +356,14 @@ public class SnappyLoader
* *
* @return the version string * @return the version string
*/ */
public static String getVersion() { public static String getVersion()
{
URL versionFile = SnappyLoader.class URL versionFile = SnappyLoader.class
.getResource("/META-INF/maven/org.xerial.snappy/snappy-java/pom.properties"); .getResource("/META-INF/maven/org.xerial.snappy/snappy-java/pom.properties");
if (versionFile == null) if (versionFile == null) {
versionFile = SnappyLoader.class.getResource("/org/xerial/snappy/VERSION"); versionFile = SnappyLoader.class.getResource("/org/xerial/snappy/VERSION");
}
String version = "unknown"; String version = "unknown";
try { try {
@ -336,8 +371,9 @@ public class SnappyLoader
Properties versionData = new Properties(); Properties versionData = new Properties();
versionData.load(versionFile.openStream()); versionData.load(versionFile.openStream());
version = versionData.getProperty("version", version); version = versionData.getProperty("version", version);
if (version.equals("unknown")) if (version.equals("unknown")) {
version = versionData.getProperty("VERSION", version); version = versionData.getProperty("VERSION", version);
}
version = version.trim().replaceAll("[^0-9M\\.]", ""); version = version.trim().replaceAll("[^0-9M\\.]", "");
} }
} }
@ -346,5 +382,4 @@ public class SnappyLoader
} }
return version; return version;
} }
} }

45
src/main/java/org/xerial/snappy/SnappyNative.java
View File

@ -30,14 +30,13 @@ import java.nio.ByteBuffer;
/** /**
* JNI interface of the {@link Snappy} implementation. The native method in this class is * JNI interface of the {@link Snappy} implementation. The native method in this class is
* defined in SnappyNative.h (genereted by javah) and SnappyNative.cpp * defined in SnappyNative.h (genereted by javah) and SnappyNative.cpp
* * <p/>
* <p> * <p>
* <b> DO NOT USE THIS CLASS since the direct use of this class might break the * <b> DO NOT USE THIS CLASS since the direct use of this class might break the
* native library code loading in {@link SnappyLoader}. </b> * native library code loading in {@link SnappyLoader}. </b>
* </p> * </p>
* *
* @author leo * @author leo
*
*/ */
public class SnappyNative public class SnappyNative
{ {
@ -47,16 +46,22 @@ public class SnappyNative
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Generic compression/decompression routines. // Generic compression/decompression routines.
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
public native long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException; public native long rawCompress(long inputAddr, long inputSize, long destAddr)
public native long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException; throws IOException;
public native long rawUncompress(long inputAddr, long inputSize, long destAddr)
throws IOException;
public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed, public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed,
int outputOffset) throws IOException; int outputOffset)
throws IOException;
public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException; public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset)
throws IOException;
public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed, public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed,
int outputOffset) throws IOException; int outputOffset)
throws IOException;
public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset) public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException; throws IOException;
@ -66,22 +71,30 @@ public class SnappyNative
public native int maxCompressedLength(int source_bytes); public native int maxCompressedLength(int source_bytes);
// This operation takes O(1) time. // This operation takes O(1) time.
public native int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException; public native int uncompressedLength(ByteBuffer compressed, int offset, int len)
throws IOException;
public native int uncompressedLength(Object input, int offset, int len) throws IOException; public native int uncompressedLength(Object input, int offset, int len)
throws IOException;
public native long uncompressedLength(long inputAddr, long len) throws IOException; public native long uncompressedLength(long inputAddr, long len)
throws IOException;
public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException; public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len)
throws IOException;
public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException; public native boolean isValidCompressedBuffer(Object input, int offset, int len)
throws IOException;
public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException; public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len)
throws IOException;
public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException; public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset)
throws IOException;
public void throw_error(int errorCode) throws IOException { public void throw_error(int errorCode)
throws IOException
{
throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode)); throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode));
} }
} }

145
src/main/java/org/xerial/snappy/SnappyOutputStream.java
View File

@ -55,7 +55,9 @@ import java.io.OutputStream;
* *
* @author leo * @author leo
*/ */
public class SnappyOutputStream extends OutputStream { public class SnappyOutputStream
extends OutputStream
{
static final int MIN_BLOCK_SIZE = 1 * 1024; static final int MIN_BLOCK_SIZE = 1 * 1024;
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
@ -65,12 +67,15 @@ public class SnappyOutputStream extends OutputStream {
private final BufferAllocator inputBufferAllocator; private final BufferAllocator inputBufferAllocator;
private final BufferAllocator outputBufferAllocator; private final BufferAllocator outputBufferAllocator;
protected final byte[] inputBuffer; // The input and output buffer fields are set to null when closing this stream:
protected final byte[] outputBuffer; protected byte[] inputBuffer;
protected byte[] outputBuffer;
private int inputCursor = 0; private int inputCursor = 0;
private int outputCursor = 0; private int outputCursor = 0;
private boolean closed;
public SnappyOutputStream(OutputStream out) { public SnappyOutputStream(OutputStream out)
{
this(out, DEFAULT_BLOCK_SIZE); this(out, DEFAULT_BLOCK_SIZE);
} }
@ -79,11 +84,13 @@ public class SnappyOutputStream extends OutputStream {
* @param blockSize byte size of the internal buffer size * @param blockSize byte size of the internal buffer size
* @throws IOException * @throws IOException
*/ */
public SnappyOutputStream(OutputStream out, int blockSize) { public SnappyOutputStream(OutputStream out, int blockSize)
this(out, blockSize, CachedBufferAllocator.factory); {
this(out, blockSize, CachedBufferAllocator.getBufferAllocatorFactory());
} }
public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactory bufferAllocatorFactory) { public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactory bufferAllocatorFactory)
{
this.out = out; this.out = out;
this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize); this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
int inputSize = blockSize; int inputSize = blockSize;
@ -98,72 +105,83 @@ public class SnappyOutputStream extends OutputStream {
outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0); outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
} }
/* (non-Javadoc) /* (non-Javadoc)
* @see java.io.OutputStream#write(byte[], int, int) * @see java.io.OutputStream#write(byte[], int, int)
*/ */
@Override @Override
public void write(byte[] b, int off, int len) throws IOException { public void write(byte[] b, int off, int len)
throws IOException
{
rawWrite(b, off, len); rawWrite(b, off, len);
} }
/** /**
* Compress the input long array data * Compress the input long array data
* *
* @param d input array * @param d input array
* @param off offset in the array * @param off offset in the array
* @param len the number of elements in the array to copy * @param len the number of elements in the array to copy
* @throws IOException * @throws IOException
*/ */
public void write(long[] d, int off, int len) throws IOException { public void write(long[] d, int off, int len)
throws IOException
{
rawWrite(d, off * 8, len * 8); rawWrite(d, off * 8, len * 8);
} }
/** /**
* Compress the input double array data * Compress the input double array data
* *
* @param f input array * @param f input array
* @param off offset in the array * @param off offset in the array
* @param len the number of elements in the array to copy * @param len the number of elements in the array to copy
* @throws IOException * @throws IOException
*/ */
public void write(double[] f, int off, int len) throws IOException { public void write(double[] f, int off, int len)
throws IOException
{
rawWrite(f, off * 8, len * 8); rawWrite(f, off * 8, len * 8);
} }
/** /**
* Compress the input float array data * Compress the input float array data
* *
* @param f input array * @param f input array
* @param off offset in the array * @param off offset in the array
* @param len the number of elements in the array to copy * @param len the number of elements in the array to copy
* @throws IOException * @throws IOException
*/ */
public void write(float[] f, int off, int len) throws IOException { public void write(float[] f, int off, int len)
throws IOException
{
rawWrite(f, off * 4, len * 4); rawWrite(f, off * 4, len * 4);
} }
/** /**
* Compress the input int array data * Compress the input int array data
* *
* @param f input array * @param f input array
* @param off offset in the array * @param off offset in the array
* @param len the number of elements in the array to copy * @param len the number of elements in the array to copy
* @throws IOException * @throws IOException
*/ */
public void write(int[] f, int off, int len) throws IOException { public void write(int[] f, int off, int len)
throws IOException
{
rawWrite(f, off * 4, len * 4); rawWrite(f, off * 4, len * 4);
} }
/** /**
* Compress the input short array data * Compress the input short array data
* *
* @param f input array * @param f input array
* @param off offset in the array * @param off offset in the array
* @param len the number of elements in the array to copy * @param len the number of elements in the array to copy
* @throws IOException * @throws IOException
*/ */
public void write(short[] f, int off, int len) throws IOException { public void write(short[] f, int off, int len)
throws IOException
{
rawWrite(f, off * 2, len * 2); rawWrite(f, off * 2, len * 2);
} }
@ -173,7 +191,9 @@ public class SnappyOutputStream extends OutputStream {
* @param d * @param d
* @throws IOException * @throws IOException
*/ */
public void write(long[] d) throws IOException { public void write(long[] d)
throws IOException
{
write(d, 0, d.length); write(d, 0, d.length);
} }
@ -183,7 +203,9 @@ public class SnappyOutputStream extends OutputStream {
* @param f * @param f
* @throws IOException * @throws IOException
*/ */
public void write(double[] f) throws IOException { public void write(double[] f)
throws IOException
{
write(f, 0, f.length); write(f, 0, f.length);
} }
@ -193,7 +215,9 @@ public class SnappyOutputStream extends OutputStream {
* @param f * @param f
* @throws IOException * @throws IOException
*/ */
public void write(float[] f) throws IOException { public void write(float[] f)
throws IOException
{
write(f, 0, f.length); write(f, 0, f.length);
} }
@ -203,7 +227,9 @@ public class SnappyOutputStream extends OutputStream {
* @param f * @param f
* @throws IOException * @throws IOException
*/ */
public void write(int[] f) throws IOException { public void write(int[] f)
throws IOException
{
write(f, 0, f.length); write(f, 0, f.length);
} }
@ -213,11 +239,14 @@ public class SnappyOutputStream extends OutputStream {
* @param f * @param f
* @throws IOException * @throws IOException
*/ */
public void write(short[] f) throws IOException { public void write(short[] f)
throws IOException
{
write(f, 0, f.length); write(f, 0, f.length);
} }
private boolean hasSufficientOutputBufferFor(int inputSize) { private boolean hasSufficientOutputBufferFor(int inputSize)
{
int maxCompressedSize = Snappy.maxCompressedLength(inputSize); int maxCompressedSize = Snappy.maxCompressedLength(inputSize);
return maxCompressedSize < outputBuffer.length - outputCursor - 4; return maxCompressedSize < outputBuffer.length - outputCursor - 4;
} }
@ -225,22 +254,28 @@ public class SnappyOutputStream extends OutputStream {
/** /**
* Compress the raw byte array data. * Compress the raw byte array data.
* *
* @param array array data of any type (e.g., byte[], float[], long[], ...) * @param array array data of any type (e.g., byte[], float[], long[], ...)
* @param byteOffset * @param byteOffset
* @param byteLength * @param byteLength
* @throws IOException * @throws IOException
*/ */
public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException { public void rawWrite(Object array, int byteOffset, int byteLength)
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
int cursor = 0; int cursor = 0;
while(cursor < byteLength) { while (cursor < byteLength) {
int readLen = Math.min(byteLength - cursor, blockSize - inputCursor); int readLen = Math.min(byteLength - cursor, blockSize - inputCursor);
// copy the input data to uncompressed buffer // copy the input data to uncompressed buffer
if(readLen > 0) { if (readLen > 0) {
Snappy.arrayCopy(array, byteOffset + cursor, readLen, inputBuffer, inputCursor); Snappy.arrayCopy(array, byteOffset + cursor, readLen, inputBuffer, inputCursor);
inputCursor += readLen; inputCursor += readLen;
} }
if(inputCursor < blockSize) if (inputCursor < blockSize) {
return; return;
}
compressInput(); compressInput();
cursor += readLen; cursor += readLen;
@ -257,8 +292,13 @@ public class SnappyOutputStream extends OutputStream {
* @see java.io.OutputStream#write(int) * @see java.io.OutputStream#write(int)
*/ */
@Override @Override
public void write(int b) throws IOException { public void write(int b)
if(inputCursor >= inputBuffer.length) { throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
if (inputCursor >= inputBuffer.length) {
compressInput(); compressInput();
} }
inputBuffer[inputCursor++] = (byte) b; inputBuffer[inputCursor++] = (byte) b;
@ -268,20 +308,27 @@ public class SnappyOutputStream extends OutputStream {
* @see java.io.OutputStream#flush() * @see java.io.OutputStream#flush()
*/ */
@Override @Override
public void flush() throws IOException { public void flush()
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
compressInput(); compressInput();
dumpOutput(); dumpOutput();
out.flush(); out.flush();
} }
static void writeInt(byte[] dst, int offset, int v) { static void writeInt(byte[] dst, int offset, int v)
{
dst[offset] = (byte) ((v >> 24) & 0xFF); dst[offset] = (byte) ((v >> 24) & 0xFF);
dst[offset + 1] = (byte) ((v >> 16) & 0xFF); dst[offset + 1] = (byte) ((v >> 16) & 0xFF);
dst[offset + 2] = (byte) ((v >> 8) & 0xFF); dst[offset + 2] = (byte) ((v >> 8) & 0xFF);
dst[offset + 3] = (byte) ((v >> 0) & 0xFF); dst[offset + 3] = (byte) ((v >> 0) & 0xFF);
} }
static int readInt(byte[] buffer, int pos) { static int readInt(byte[] buffer, int pos)
{
int b1 = (buffer[pos] & 0xFF) << 24; int b1 = (buffer[pos] & 0xFF) << 24;
int b2 = (buffer[pos + 1] & 0xFF) << 16; int b2 = (buffer[pos + 1] & 0xFF) << 16;
int b3 = (buffer[pos + 2] & 0xFF) << 8; int b3 = (buffer[pos + 2] & 0xFF) << 8;
@ -289,20 +336,24 @@ public class SnappyOutputStream extends OutputStream {
return b1 | b2 | b3 | b4; return b1 | b2 | b3 | b4;
} }
protected void dumpOutput() throws IOException { protected void dumpOutput()
if(outputCursor > 0) { throws IOException
{
if (outputCursor > 0) {
out.write(outputBuffer, 0, outputCursor); out.write(outputBuffer, 0, outputCursor);
outputCursor = 0; outputCursor = 0;
} }
} }
protected void compressInput() throws IOException { protected void compressInput()
if(inputCursor <= 0) { throws IOException
{
if (inputCursor <= 0) {
return; // no need to dump return; // no need to dump
} }
// Compress and dump the buffer content // Compress and dump the buffer content
if(!hasSufficientOutputBufferFor(inputCursor)) { if (!hasSufficientOutputBufferFor(inputCursor)) {
dumpOutput(); dumpOutput();
} }
int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4); int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4);
@ -319,14 +370,22 @@ public class SnappyOutputStream extends OutputStream {
* @see java.io.OutputStream#close() * @see java.io.OutputStream#close()
*/ */
@Override @Override
public void close() throws IOException { public void close()
throws IOException
{
if (closed) {
return;
}
try { try {
flush(); flush();
out.close(); out.close();
} finally { }
finally {
closed = true;
inputBufferAllocator.release(inputBuffer); inputBufferAllocator.release(inputBuffer);
outputBufferAllocator.release(outputBuffer); outputBufferAllocator.release(outputBuffer);
inputBuffer = null;
outputBuffer = null;
} }
} }
} }

5
src/main/java/org/xerial/snappy/buffer/BufferAllocator.java
View File

@ -3,9 +3,10 @@ package org.xerial.snappy.buffer;
/** /**
* BufferAllocator interface. The implementation of this interface must be thread-safe * BufferAllocator interface. The implementation of this interface must be thread-safe
*/ */
public interface BufferAllocator { public interface BufferAllocator
{
public byte[] allocate(int size); public byte[] allocate(int size);
public void release(byte[] buffer);
public void release(byte[] buffer);
} }

3
src/main/java/org/xerial/snappy/buffer/BufferAllocatorFactory.java
View File

@ -3,7 +3,8 @@ package org.xerial.snappy.buffer;
/** /**
* *
*/ */
public interface BufferAllocatorFactory { public interface BufferAllocatorFactory
{
BufferAllocator getBufferAllocator(int minSize); BufferAllocator getBufferAllocator(int minSize);
} }

43
src/main/java/org/xerial/snappy/buffer/CachedBufferAllocator.java
View File

@ -6,29 +6,45 @@ import java.util.*;
/** /**
* Cached buffer * Cached buffer
*/ */
public class CachedBufferAllocator implements BufferAllocator { public class CachedBufferAllocator
implements BufferAllocator
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() { {
private static BufferAllocatorFactory factory = new BufferAllocatorFactory()
{
@Override @Override
public BufferAllocator getBufferAllocator(int bufferSize) { public BufferAllocator getBufferAllocator(int bufferSize)
{
return CachedBufferAllocator.getAllocator(bufferSize); return CachedBufferAllocator.getAllocator(bufferSize);
} }
}; };
public static void setBufferAllocatorFactory(BufferAllocatorFactory factory)
{
assert (factory != null);
CachedBufferAllocator.factory = factory;
}
public static BufferAllocatorFactory getBufferAllocatorFactory()
{
return factory;
}
/** /**
* Use SoftReference so that having this queueTable does not prevent the GC of CachedBufferAllocator instances * Use SoftReference so that having this queueTable does not prevent the GC of CachedBufferAllocator instances
*/ */
public static Map<Integer, SoftReference<CachedBufferAllocator>> queueTable = new HashMap<Integer, SoftReference<CachedBufferAllocator>>(); private static final Map<Integer, SoftReference<CachedBufferAllocator>> queueTable = new HashMap<Integer, SoftReference<CachedBufferAllocator>>();
private final int bufferSize; private final int bufferSize;
private final Deque<byte[]> bufferQueue; private final Deque<byte[]> bufferQueue;
public CachedBufferAllocator(int bufferSize) { public CachedBufferAllocator(int bufferSize)
{
this.bufferSize = bufferSize; this.bufferSize = bufferSize;
this.bufferQueue = new ArrayDeque<byte[]>(); this.bufferQueue = new ArrayDeque<byte[]>();
} }
public static synchronized CachedBufferAllocator getAllocator(int bufferSize) { public static synchronized CachedBufferAllocator getAllocator(int bufferSize)
{
CachedBufferAllocator result = null; CachedBufferAllocator result = null;
if (queueTable.containsKey(bufferSize)) { if (queueTable.containsKey(bufferSize)) {
@ -42,9 +58,10 @@ public class CachedBufferAllocator implements BufferAllocator {
} }
@Override @Override
public byte[] allocate(int size) { public byte[] allocate(int size)
synchronized(this) { {
if(bufferQueue.isEmpty()) { synchronized (this) {
if (bufferQueue.isEmpty()) {
return new byte[size]; return new byte[size];
} }
else { else {
@ -52,9 +69,11 @@ public class CachedBufferAllocator implements BufferAllocator {
} }
} }
} }
@Override @Override
public void release(byte[] buffer) { public void release(byte[] buffer)
synchronized(this) { {
synchronized (this) {
bufferQueue.addLast(buffer); bufferQueue.addLast(buffer);
} }
} }

18
src/main/java/org/xerial/snappy/buffer/DefaultBufferAllocator.java
View File

@ -3,24 +3,30 @@ package org.xerial.snappy.buffer;
/** /**
* Simple buffer allocator, which does not reuse the allocated buffer * Simple buffer allocator, which does not reuse the allocated buffer
*/ */
public class DefaultBufferAllocator implements BufferAllocator { public class DefaultBufferAllocator
implements BufferAllocator
{
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() { public static BufferAllocatorFactory factory = new BufferAllocatorFactory()
{
public BufferAllocator singleton = new DefaultBufferAllocator(); public BufferAllocator singleton = new DefaultBufferAllocator();
@Override @Override
public BufferAllocator getBufferAllocator(int bufferSize) { public BufferAllocator getBufferAllocator(int bufferSize)
{
return singleton; return singleton;
} }
}; };
@Override @Override
public byte[] allocate(int size) { public byte[] allocate(int size)
{
return new byte[size]; return new byte[size];
} }
@Override @Override
public void release(byte[] buffer) { public void release(byte[] buffer)
{
// do nothing // do nothing
} }
} }

6
src/main/java/org/xerial/snappy/package-info.java
View File

@ -16,7 +16,7 @@
/** /**
* Snappy API for compressing/decompressing data. * Snappy API for compressing/decompressing data.
* * <p/>
* Usage * Usage
* First, import {@link org.xerial.snappy.Snappy} in your Java code: * First, import {@link org.xerial.snappy.Snappy} in your Java code:
* <code> * <code>
@ -34,9 +34,9 @@
* System.out.println(result); * System.out.println(result);
* </pre> * </pre>
* </code> * </code>
* * <p/>
* <p>In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used. </p> * <p>In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used. </p>
* * <p/>
* <h3>Stream-based API</h3> * <h3>Stream-based API</h3>
* Stream-based compressor/decompressor SnappyOutputStream, SnappyInputStream are also available for reading/writing large data sets. * Stream-based compressor/decompressor SnappyOutputStream, SnappyInputStream are also available for reading/writing large data sets.
*/ */

2
src/main/resources/org/xerial/snappy/VERSION
View File

@ -1 +1 @@
VERSION=1.1.2 VERSION=1.1.3

BIN
src/main/resources/org/xerial/snappy/native/AIX/ppc/libsnappyjava.a Executable file
View File

Binary file not shown.

BIN
src/main/resources/org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a
View File

Binary file not shown.

BIN
src/main/resources/org/xerial/snappy/native/Linux/s390x/libsnappyjava.so Executable file
View File

Binary file not shown.

68
src/test/java/org/xerial/snappy/CalgaryTest.java
View File

@ -45,7 +45,6 @@ import org.xerial.util.log.Logger;
* Benchmark using Calgary data set * Benchmark using Calgary data set
* *
* @author leo * @author leo
*
*/ */
public class CalgaryTest public class CalgaryTest
{ {
@ -54,24 +53,30 @@ public class CalgaryTest
@Rule @Rule
public final TemporaryFolder tempFolder = new TemporaryFolder(); public final TemporaryFolder tempFolder = new TemporaryFolder();
static byte[] readFile(String file) throws IOException { static byte[] readFile(String file)
throws IOException
{
InputStream in = FileResource.find(CalgaryTest.class, file).openStream(); InputStream in = FileResource.find(CalgaryTest.class, file).openStream();
if (in == null) if (in == null) {
throw new IOException("file " + file + " is not found"); throw new IOException("file " + file + " is not found");
}
try { try {
return SnappyInputStreamTest.readFully(in); return SnappyInputStreamTest.readFully(in);
} }
finally { finally {
if (in != null) if (in != null) {
in.close(); in.close();
}
} }
} }
public static final String[] files = { "bib", "book1", "book2", "geo", "news", "obj1", "obj2", "paper1", "paper2", public static final String[] files = {"bib", "book1", "book2", "geo", "news", "obj1", "obj2", "paper1", "paper2",
"paper3", "paper4", "paper5", "paper6", "pic", "progc", "progl", "progp", "trans" }; "paper3", "paper4", "paper5", "paper6", "pic", "progc", "progl", "progp", "trans"};
@Test @Test
public void block() throws Exception { public void block()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
@ -83,7 +88,9 @@ public class CalgaryTest
} }
@Test @Test
public void stream() throws Exception { public void stream()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
@ -101,7 +108,9 @@ public class CalgaryTest
} }
@Test @Test
public void streamFramed() throws Exception { public void streamFramed()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
@ -121,35 +130,33 @@ public class CalgaryTest
} }
@Test @Test
public void streamFramedToFile() throws Exception { public void streamFramedToFile()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
final File tempFile = tempFolder.newFile(f); final File tempFile = tempFolder.newFile(f);
final FileOutputStream compressedFOS = new FileOutputStream(tempFile); final FileOutputStream compressedFOS = new FileOutputStream(tempFile);
try try {
{
SnappyFramedOutputStream out = new SnappyFramedOutputStream(compressedFOS); SnappyFramedOutputStream out = new SnappyFramedOutputStream(compressedFOS);
out.write(orig); out.write(orig);
out.close(); out.close();
} }
finally finally {
{
compressedFOS.close(); compressedFOS.close();
} }
byte[] uncompressed = new byte[orig.length]; byte[] uncompressed = new byte[orig.length];
final FileInputStream compressedFIS = new FileInputStream(tempFile); final FileInputStream compressedFIS = new FileInputStream(tempFile);
try try {
{
SnappyFramedInputStream in = new SnappyFramedInputStream(compressedFIS.getChannel()); SnappyFramedInputStream in = new SnappyFramedInputStream(compressedFIS.getChannel());
int readBytes = readBytes(in, uncompressed, 0, orig.length); int readBytes = readBytes(in, uncompressed, 0, orig.length);
assertEquals(orig.length, readBytes); assertEquals(orig.length, readBytes);
} }
finally finally {
{
compressedFIS.close(); compressedFIS.close();
} }
@ -158,7 +165,9 @@ public class CalgaryTest
} }
@Test @Test
public void streamFramedNoCRCVerify() throws Exception { public void streamFramedNoCRCVerify()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
@ -178,7 +187,9 @@ public class CalgaryTest
} }
@Test @Test
public void byteWiseRead() throws Exception { public void byteWiseRead()
throws Exception
{
for (String f : files) { for (String f : files) {
byte[] orig = readFile("testdata/calgary/" + f); byte[] orig = readFile("testdata/calgary/" + f);
@ -190,10 +201,11 @@ public class CalgaryTest
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressedBuf.toByteArray())); SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressedBuf.toByteArray()));
byte[] uncompressed = new byte[orig.length]; byte[] uncompressed = new byte[orig.length];
int cursor = 0; int cursor = 0;
for (;;) { for (; ; ) {
int b = in.read(); int b = in.read();
if (b == -1) if (b == -1) {
break; break;
}
uncompressed[cursor++] = (byte) b; uncompressed[cursor++] = (byte) b;
} }
assertEquals(orig.length, cursor); assertEquals(orig.length, cursor);
@ -201,7 +213,8 @@ public class CalgaryTest
} }
} }
static final int readBytes(InputStream source, byte[] dest, int offset, int length) throws IOException static final int readBytes(InputStream source, byte[] dest, int offset, int length)
throws IOException
{ {
// how many bytes were read. // how many bytes were read.
int lastRead = source.read(dest, offset, length); int lastRead = source.read(dest, offset, length);
@ -209,16 +222,13 @@ public class CalgaryTest
int totalRead = lastRead; int totalRead = lastRead;
// if we did not read as many bytes as we had hoped, try reading again. // if we did not read as many bytes as we had hoped, try reading again.
if (lastRead < length) if (lastRead < length) {
{
// as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading. // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
while (totalRead < length && lastRead != -1) while (totalRead < length && lastRead != -1) {
{
lastRead = source.read(dest, offset + totalRead, length - totalRead); lastRead = source.read(dest, offset + totalRead, length - totalRead);
// if we got EOF, do not add to total read. // if we got EOF, do not add to total read.
if (lastRead != -1) if (lastRead != -1) {
{
totalRead += lastRead; totalRead += lastRead;
} }
} }

23
src/test/java/org/xerial/snappy/OSInfoTest.java
View File

@ -34,7 +34,8 @@ import org.junit.Test;
public class OSInfoTest public class OSInfoTest
{ {
@Test @Test
public void osName() { public void osName()
{
assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows XP")); assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows XP"));
assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows 2000")); assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows 2000"));
assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows Vista")); assertEquals("Windows", OSInfo.translateOSNameToFolderName("Windows Vista"));
@ -53,7 +54,8 @@ public class OSInfoTest
} }
@Test @Test
public void archName() { public void archName()
{
assertEquals("i386", OSInfo.translateArchNameToFolderName("i386")); assertEquals("i386", OSInfo.translateArchNameToFolderName("i386"));
assertEquals("x86", OSInfo.translateArchNameToFolderName("x86")); assertEquals("x86", OSInfo.translateArchNameToFolderName("x86"));
assertEquals("ppc", OSInfo.translateArchNameToFolderName("ppc")); assertEquals("ppc", OSInfo.translateArchNameToFolderName("ppc"));
@ -61,7 +63,8 @@ public class OSInfoTest
} }
@Test @Test
public void folderPath() { public void folderPath()
{
String[] component = OSInfo.getNativeLibFolderPathForCurrentOS().split("/"); String[] component = OSInfo.getNativeLibFolderPathForCurrentOS().split("/");
assertEquals(2, component.length); assertEquals(2, component.length);
assertEquals(OSInfo.getOSName(), component[0]); assertEquals(OSInfo.getOSName(), component[0]);
@ -69,7 +72,9 @@ public class OSInfoTest
} }
@Test @Test
public void testMainForOSName() throws Exception { public void testMainForOSName()
throws Exception
{
// preserve the current System.out // preserve the current System.out
PrintStream out = System.out; PrintStream out = System.out;
@ -78,18 +83,19 @@ public class OSInfoTest
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
PrintStream tmpOut = new PrintStream(buf); PrintStream tmpOut = new PrintStream(buf);
System.setOut(tmpOut); System.setOut(tmpOut);
OSInfo.main(new String[] { "--os" }); OSInfo.main(new String[] {"--os"});
assertEquals(OSInfo.getOSName(), buf.toString()); assertEquals(OSInfo.getOSName(), buf.toString());
} }
finally { finally {
// reset STDOUT // reset STDOUT
System.setOut(out); System.setOut(out);
} }
} }
@Test @Test
public void testMainForArchName() throws Exception { public void testMainForArchName()
throws Exception
{
// preserver the current System.out // preserver the current System.out
PrintStream out = System.out; PrintStream out = System.out;
@ -98,7 +104,7 @@ public class OSInfoTest
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
PrintStream tmpOut = new PrintStream(buf); PrintStream tmpOut = new PrintStream(buf);
System.setOut(tmpOut); System.setOut(tmpOut);
OSInfo.main(new String[] { "--arch" }); OSInfo.main(new String[] {"--arch"});
assertEquals(OSInfo.getArchName(), buf.toString()); assertEquals(OSInfo.getArchName(), buf.toString());
} }
finally { finally {
@ -106,5 +112,4 @@ public class OSInfoTest
System.setOut(out); System.setOut(out);
} }
} }
} }

19
src/test/java/org/xerial/snappy/RandomGenerator.java
View File

@ -7,17 +7,19 @@ import java.util.Random;
/** /**
* Generates random data with specific expected snappy performance characteristics. * Generates random data with specific expected snappy performance characteristics.
* * <p/>
* <p> * <p>
* This has been copied from <a href="https://github.com/dain/snappy"> dain's snappy</a> implementation.. * This has been copied from <a href="https://github.com/dain/snappy"> dain's snappy</a> implementation..
* </p> * </p>
*/ */
public class RandomGenerator { public class RandomGenerator
{
public final byte[] data; public final byte[] data;
public int position; public int position;
public RandomGenerator(double compressionRatio) { public RandomGenerator(double compressionRatio)
{
// We use a limited amount of data over and over again and ensure // We use a limited amount of data over and over again and ensure
// that it is larger than the compression window (32KB), and also // that it is larger than the compression window (32KB), and also
// large enough to serve all typical value sizes we want to write. // large enough to serve all typical value sizes we want to write.
@ -30,7 +32,8 @@ public class RandomGenerator {
} }
} }
public int getNextPosition(int length) { public int getNextPosition(int length)
{
if (position + length > data.length) { if (position + length > data.length) {
position = 0; position = 0;
assert (length < data.length); assert (length < data.length);
@ -41,7 +44,8 @@ public class RandomGenerator {
} }
private static byte[] compressibleData(Random random, private static byte[] compressibleData(Random random,
double compressionRatio, int length) { double compressionRatio, int length)
{
int raw = (int) (length * compressionRatio); int raw = (int) (length * compressionRatio);
if (raw < 1) { if (raw < 1) {
raw = 1; raw = 1;
@ -50,7 +54,7 @@ public class RandomGenerator {
// Duplicate the random data until we have filled "length" bytes // Duplicate the random data until we have filled "length" bytes
byte[] dest = new byte[length]; byte[] dest = new byte[length];
for (int i = 0; i < length;) { for (int i = 0; i < length; ) {
int chunkLength = Math.min(rawData.length, length - i); int chunkLength = Math.min(rawData.length, length - i);
System.arraycopy(rawData, 0, dest, i, chunkLength); System.arraycopy(rawData, 0, dest, i, chunkLength);
i += chunkLength; i += chunkLength;
@ -58,7 +62,8 @@ public class RandomGenerator {
return dest; return dest;
} }
private static byte[] generateRandomData(Random random, int length) { private static byte[] generateRandomData(Random random, int length)
{
byte[] rawData = new byte[length]; byte[] rawData = new byte[length];
for (int i = 0; i < rawData.length; i++) { for (int i = 0; i < rawData.length; i++) {
rawData[i] = (byte) random.nextInt(256); rawData[i] = (byte) random.nextInt(256);

132
src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java
View File

@ -29,13 +29,15 @@ import org.junit.Test;
* *
* @author Brett Okken * @author Brett Okken
*/ */
public class SnappyFramedStreamTest { public class SnappyFramedStreamTest
{
/** /**
* @throws IOException * @throws IOException
*/ */
protected OutputStream createOutputStream(OutputStream target) protected OutputStream createOutputStream(OutputStream target)
throws IOException { throws IOException
{
return new SnappyFramedOutputStream(target); return new SnappyFramedOutputStream(target);
} }
@ -45,16 +47,21 @@ public class SnappyFramedStreamTest {
* @throws IOException * @throws IOException
*/ */
protected InputStream createInputStream(InputStream source, protected InputStream createInputStream(InputStream source,
boolean verifyCheckSums) throws IOException { boolean verifyCheckSums)
throws IOException
{
return new SnappyFramedInputStream(source, verifyCheckSums); return new SnappyFramedInputStream(source, verifyCheckSums);
} }
protected byte[] getMarkerFrame() { protected byte[] getMarkerFrame()
{
return HEADER_BYTES; return HEADER_BYTES;
} }
@Test @Test
public void testSimple() throws Exception { public void testSimple()
throws Exception
{
byte[] original = "aaaaaaaaaaaabbbbbbbaaaaaa".getBytes("utf-8"); byte[] original = "aaaaaaaaaaaabbbbbbbaaaaaa".getBytes("utf-8");
byte[] compressed = compress(original); byte[] compressed = compress(original);
@ -83,7 +90,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testUncompressable() throws Exception { public void testUncompressable()
throws Exception
{
byte[] random = getRandom(1, 5000); byte[] random = getRandom(1, 5000);
int crc32c = maskedCrc32c(random); int crc32c = maskedCrc32c(random);
@ -103,71 +112,90 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testEmptyCompression() throws Exception { public void testEmptyCompression()
throws Exception
{
byte[] empty = new byte[0]; byte[] empty = new byte[0];
assertArrayEquals(compress(empty), HEADER_BYTES); assertArrayEquals(compress(empty), HEADER_BYTES);
assertArrayEquals(uncompress(HEADER_BYTES), empty); assertArrayEquals(uncompress(HEADER_BYTES), empty);
} }
@Test(expected = EOFException.class) @Test(expected = EOFException.class)
public void testShortBlockHeader() throws Exception { public void testShortBlockHeader()
uncompressBlock(new byte[] { 0 }); throws Exception
{
uncompressBlock(new byte[] {0});
} }
@Test(expected = EOFException.class) @Test(expected = EOFException.class)
public void testShortBlockData() throws Exception { public void testShortBlockData()
throws Exception
{
// flag = 0, size = 8, crc32c = 0, block data= [x, x] // flag = 0, size = 8, crc32c = 0, block data= [x, x]
uncompressBlock(new byte[] { 1, 8, 0, 0, 0, 0, 0, 0, 'x', 'x' }); uncompressBlock(new byte[] {1, 8, 0, 0, 0, 0, 0, 0, 'x', 'x'});
} }
@Test @Test
public void testUnskippableChunkFlags() throws Exception { public void testUnskippableChunkFlags()
throws Exception
{
for (int i = 2; i <= 0x7f; i++) { for (int i = 2; i <= 0x7f; i++) {
try { try {
uncompressBlock(new byte[] { (byte) i, 5, 0, 0, 0, 0, 0, 0, 0 }); uncompressBlock(new byte[] {(byte) i, 5, 0, 0, 0, 0, 0, 0, 0});
fail("no exception thrown with flag: " + Integer.toHexString(i)); fail("no exception thrown with flag: " + Integer.toHexString(i));
} catch (IOException e) { }
catch (IOException e) {
} }
} }
} }
@Test @Test
public void testSkippableChunkFlags() throws Exception { public void testSkippableChunkFlags()
throws Exception
{
for (int i = 0x80; i <= 0xfe; i++) { for (int i = 0x80; i <= 0xfe; i++) {
try { try {
uncompressBlock(new byte[] { (byte) i, 5, 0, 0, 0, 0, 0, 0, 0 }); uncompressBlock(new byte[] {(byte) i, 5, 0, 0, 0, 0, 0, 0, 0});
} catch (IOException e) { }
catch (IOException e) {
fail("exception thrown with flag: " + Integer.toHexString(i)); fail("exception thrown with flag: " + Integer.toHexString(i));
} }
} }
} }
@Test(expected = IOException.class) @Test(expected = IOException.class)
public void testInvalidBlockSizeZero() throws Exception { public void testInvalidBlockSizeZero()
throws Exception
{
// flag = '0', block size = 4, crc32c = 0 // flag = '0', block size = 4, crc32c = 0
uncompressBlock(new byte[] { 1, 4, 0, 0, 0, 0, 0, 0 }); uncompressBlock(new byte[] {1, 4, 0, 0, 0, 0, 0, 0});
} }
@Test(expected = IOException.class) @Test(expected = IOException.class)
public void testInvalidChecksum() throws Exception { public void testInvalidChecksum()
throws Exception
{
// flag = 0, size = 5, crc32c = 0, block data = [a] // flag = 0, size = 5, crc32c = 0, block data = [a]
uncompressBlock(new byte[] { 1, 5, 0, 0, 0, 0, 0, 0, 'a' }); uncompressBlock(new byte[] {1, 5, 0, 0, 0, 0, 0, 0, 'a'});
} }
@Test @Test
public void testInvalidChecksumIgnoredWhenVerificationDisabled() public void testInvalidChecksumIgnoredWhenVerificationDisabled()
throws Exception { throws Exception
{
// flag = 0, size = 4, crc32c = 0, block data = [a] // flag = 0, size = 4, crc32c = 0, block data = [a]
byte[] block = { 1, 5, 0, 0, 0, 0, 0, 0, 'a' }; byte[] block = {1, 5, 0, 0, 0, 0, 0, 0, 'a'};
ByteArrayInputStream inputData = new ByteArrayInputStream( ByteArrayInputStream inputData = new ByteArrayInputStream(
blockToStream(block)); blockToStream(block));
assertArrayEquals(toByteArray(createInputStream(inputData, false)), assertArrayEquals(toByteArray(createInputStream(inputData, false)),
new byte[] { 'a' }); new byte[] {'a'});
} }
@Test @Test
public void testTransferFrom_InputStream() throws IOException { public void testTransferFrom_InputStream()
throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final ByteArrayOutputStream baos = new ByteArrayOutputStream( final ByteArrayOutputStream baos = new ByteArrayOutputStream(
@ -184,7 +212,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testTransferFrom_ReadableByteChannel() throws IOException { public void testTransferFrom_ReadableByteChannel()
throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final ByteArrayOutputStream baos = new ByteArrayOutputStream( final ByteArrayOutputStream baos = new ByteArrayOutputStream(
@ -201,7 +231,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testTransferTo_OutputStream() throws IOException { public void testTransferTo_OutputStream()
throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final byte[] compressed = compress(random); final byte[] compressed = compress(random);
@ -216,7 +248,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testTransferTo_WritableByteChannel() throws IOException { public void testTransferTo_WritableByteChannel()
throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final byte[] compressed = compress(random); final byte[] compressed = compress(random);
@ -233,7 +267,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testLargerFrames_raw_() throws IOException { public void testLargerFrames_raw_()
throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final byte[] stream = new byte[HEADER_BYTES.length + 8 + random.length]; final byte[] stream = new byte[HEADER_BYTES.length + 8 + random.length];
@ -260,7 +296,9 @@ public class SnappyFramedStreamTest {
} }
@Test @Test
public void testLargerFrames_compressed_() throws IOException { public void testLargerFrames_compressed_()
throws IOException
{
final byte[] random = getRandom(0.5, 500000); final byte[] random = getRandom(0.5, 500000);
final byte[] compressed = Snappy.compress(random); final byte[] compressed = Snappy.compress(random);
@ -290,7 +328,8 @@ public class SnappyFramedStreamTest {
@Test @Test
public void testLargerFrames_compressed_smaller_raw_larger() public void testLargerFrames_compressed_smaller_raw_larger()
throws IOException { throws IOException
{
final byte[] random = getRandom(0.5, 100000); final byte[] random = getRandom(0.5, 100000);
final byte[] compressed = Snappy.compress(random); final byte[] compressed = Snappy.compress(random);
@ -319,19 +358,23 @@ public class SnappyFramedStreamTest {
assertArrayEquals(random, uncompressed); assertArrayEquals(random, uncompressed);
} }
private byte[] uncompressBlock(byte[] block) throws IOException { private byte[] uncompressBlock(byte[] block)
throws IOException
{
return uncompress(blockToStream(block)); return uncompress(blockToStream(block));
} }
private static byte[] blockToStream(byte[] block) { private static byte[] blockToStream(byte[] block)
{
byte[] stream = new byte[HEADER_BYTES.length + block.length]; byte[] stream = new byte[HEADER_BYTES.length + block.length];
System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length);
System.arraycopy(block, 0, stream, HEADER_BYTES.length, block.length); System.arraycopy(block, 0, stream, HEADER_BYTES.length, block.length);
return stream; return stream;
} }
protected byte[] compress(byte[] original)
protected byte[] compress(byte[] original) throws IOException { throws IOException
{
ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream();
OutputStream snappyOut = createOutputStream(out); OutputStream snappyOut = createOutputStream(out);
snappyOut.write(original); snappyOut.write(original);
@ -339,29 +382,35 @@ public class SnappyFramedStreamTest {
return out.toByteArray(); return out.toByteArray();
} }
protected byte[] uncompress(byte[] compressed) throws IOException { protected byte[] uncompress(byte[] compressed)
return toByteArray(createInputStream(new ByteArrayInputStream( throws IOException
{
return toByteArray(createInputStream(new ByteArrayInputStream(
compressed), true)); compressed), true));
} }
private static byte[] toByteArray(InputStream createInputStream) throws IOException { private static byte[] toByteArray(InputStream createInputStream)
throws IOException
{
final ByteArrayOutputStream baos = new ByteArrayOutputStream(64 * 1024); final ByteArrayOutputStream baos = new ByteArrayOutputStream(64 * 1024);
final byte[] buffer = new byte[8 * 1024]; final byte[] buffer = new byte[8 * 1024];
int read; int read;
while((read = createInputStream.read(buffer)) > 0) { while ((read = createInputStream.read(buffer)) > 0) {
baos.write(buffer, 0, read); baos.write(buffer, 0, read);
} }
return baos.toByteArray(); return baos.toByteArray();
} }
static int toInt(byte value) { static int toInt(byte value)
{
return value & 0xFF; return value & 0xFF;
} }
private byte[] getRandom(double compressionRatio, int length) { private byte[] getRandom(double compressionRatio, int length)
{
RandomGenerator gen = new RandomGenerator( RandomGenerator gen = new RandomGenerator(
compressionRatio); compressionRatio);
gen.getNextPosition(length); gen.getNextPosition(length);
@ -369,5 +418,4 @@ public class SnappyFramedStreamTest {
assertEquals(random.length, length); assertEquals(random.length, length);
return random; return random;
} }
} }

62
src/test/java/org/xerial/snappy/SnappyInputStreamTest.java
View File

@ -35,24 +35,27 @@ import java.io.InputStream;
import org.junit.Test; import org.junit.Test;
import org.xerial.util.FileResource; import org.xerial.util.FileResource;
import org.xerial.util.log.Logger; import org.xerial.util.log.Logger;
import scala.Array;
public class SnappyInputStreamTest public class SnappyInputStreamTest
{ {
private static Logger _logger = Logger.getLogger(SnappyInputStreamTest.class); private static Logger _logger = Logger.getLogger(SnappyInputStreamTest.class);
public static byte[] readResourceFile(String fileName) throws IOException { public static byte[] readResourceFile(String fileName)
throws IOException
{
BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class, fileName) BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class, fileName)
.openStream()); .openStream());
assertNotNull(input); assertNotNull(input);
return readFully(input); return readFully(input);
} }
public static byte[] readFully(InputStream input) throws IOException { public static byte[] readFully(InputStream input)
throws IOException
{
try { try {
ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buf = new byte[4096]; byte[] buf = new byte[4096];
for (int readBytes = 0; (readBytes = input.read(buf)) != -1;) { for (int readBytes = 0; (readBytes = input.read(buf)) != -1; ) {
out.write(buf, 0, readBytes); out.write(buf, 0, readBytes);
} }
out.flush(); out.flush();
@ -63,10 +66,12 @@ public class SnappyInputStreamTest
} }
} }
public static byte[] byteWiseReadFully(InputStream input) throws IOException { public static byte[] byteWiseReadFully(InputStream input)
throws IOException
{
ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buf = new byte[4096]; byte[] buf = new byte[4096];
for (int readData = 0; (readData = input.read()) != -1;) { for (int readData = 0; (readData = input.read()) != -1; ) {
out.write(readData); out.write(readData);
} }
out.flush(); out.flush();
@ -74,7 +79,9 @@ public class SnappyInputStreamTest
} }
@Test @Test
public void read() throws Exception { public void read()
throws Exception
{
ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream();
SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf);
byte[] orig = readResourceFile("alice29.txt"); byte[] orig = readResourceFile("alice29.txt");
@ -88,11 +95,12 @@ public class SnappyInputStreamTest
assertEquals(orig.length, uncompressed.length); assertEquals(orig.length, uncompressed.length);
assertArrayEquals(orig, uncompressed); assertArrayEquals(orig, uncompressed);
} }
@Test @Test
public void readBlockCompressedData() throws Exception { public void readBlockCompressedData()
throws Exception
{
byte[] orig = readResourceFile("alice29.txt"); byte[] orig = readResourceFile("alice29.txt");
byte[] compressed = Snappy.compress(orig); byte[] compressed = Snappy.compress(orig);
@ -104,7 +112,9 @@ public class SnappyInputStreamTest
} }
@Test @Test
public void biteWiseRead() throws Exception { public void biteWiseRead()
throws Exception
{
byte[] orig = readResourceFile("testdata/calgary/paper6"); byte[] orig = readResourceFile("testdata/calgary/paper6");
byte[] compressed = Snappy.compress(orig); byte[] compressed = Snappy.compress(orig);
@ -113,17 +123,18 @@ public class SnappyInputStreamTest
assertEquals(orig.length, uncompressed.length); assertEquals(orig.length, uncompressed.length);
assertArrayEquals(orig, uncompressed); assertArrayEquals(orig, uncompressed);
} }
@Test @Test
public void available() throws Exception { public void available()
throws Exception
{
byte[] orig = readResourceFile("testdata/calgary/paper6"); byte[] orig = readResourceFile("testdata/calgary/paper6");
byte[] compressed = Snappy.compress(orig); byte[] compressed = Snappy.compress(orig);
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed)); SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed));
byte[] buf = new byte[4]; byte[] buf = new byte[4];
for (int readBytes = 0; (readBytes = in.read(buf)) != -1;) { for (int readBytes = 0; (readBytes = in.read(buf)) != -1; ) {
assertTrue(in.available() >= 0); assertTrue(in.available() >= 0);
} }
assertTrue(in.available() == 0); assertTrue(in.available() == 0);
@ -131,19 +142,23 @@ public class SnappyInputStreamTest
} }
@Test @Test
public void emptyStream() throws Exception { public void emptyStream()
throws Exception
{
try { try {
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(new byte[0])); SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(new byte[0]));
byte[] uncompressed = readFully(in); byte[] uncompressed = readFully(in);
assertEquals(0, uncompressed.length); assertEquals(0, uncompressed.length);
fail("should not reach here"); fail("should not reach here");
} }
catch(SnappyIOException e) { catch (SnappyIOException e) {
assertEquals(SnappyErrorCode.EMPTY_INPUT, e.getErrorCode()); assertEquals(SnappyErrorCode.EMPTY_INPUT, e.getErrorCode());
} }
} }
public static byte[] compressResource(String resourcePath) throws Exception { public static byte[] compressResource(String resourcePath)
throws Exception
{
ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream(); ByteArrayOutputStream compressedBuf = new ByteArrayOutputStream();
SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf); SnappyOutputStream snappyOut = new SnappyOutputStream(compressedBuf);
byte[] orig = readResourceFile(resourcePath); byte[] orig = readResourceFile(resourcePath);
@ -153,7 +168,9 @@ public class SnappyInputStreamTest
} }
@Test @Test
public void chunkRead() throws Exception { public void chunkRead()
throws Exception
{
byte[] chunk1 = compressResource("alice29.txt"); byte[] chunk1 = compressResource("alice29.txt");
byte[] chunk2 = compressResource("testdata/calgary/paper6"); byte[] chunk2 = compressResource("testdata/calgary/paper6");
@ -176,4 +193,15 @@ public class SnappyInputStreamTest
assertArrayEquals(orig2, uncompressed2); assertArrayEquals(orig2, uncompressed2);
} }
@Test
public void readSnappyCompressResult()
throws Exception
{
byte[] orig = readResourceFile("alice29.txt");
byte[] compressed = Snappy.compress(orig);
SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(compressed));
byte[] uncompressed = readFully(in);
assertArrayEquals(orig, uncompressed);
}
} }

40
src/test/java/org/xerial/snappy/SnappyLoaderTest.java
View File

@ -42,26 +42,31 @@ public class SnappyLoaderTest
{ {
private static Logger _logger = Logger.getLogger(SnappyLoaderTest.class); private static Logger _logger = Logger.getLogger(SnappyLoaderTest.class);
public static BufferedInputStream openByteStream(Class< ? > referenceClass, String resourceFileName) public static BufferedInputStream openByteStream(Class<?> referenceClass, String resourceFileName)
throws IOException { throws IOException
{
URL url = FileResource.find(referenceClass, resourceFileName); URL url = FileResource.find(referenceClass, resourceFileName);
if (url != null) { if (url != null) {
return new BufferedInputStream(url.openStream()); return new BufferedInputStream(url.openStream());
} }
else else {
return null; return null;
}
} }
public static <T> String loadIntoString(Class<T> referenceClass, String path) throws IOException { public static <T> String loadIntoString(Class<T> referenceClass, String path)
throws IOException
{
BufferedInputStream in = openByteStream(referenceClass, path); BufferedInputStream in = openByteStream(referenceClass, path);
if (in == null) if (in == null) {
throw new FileNotFoundException( throw new FileNotFoundException(
String.format("reference class:%s, path:%s", referenceClass.getName(), path)); String.format("reference class:%s, path:%s", referenceClass.getName(), path));
}
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
try { try {
byte[] tmp = new byte[4028]; byte[] tmp = new byte[4028];
for (int readBytes = 0; (readBytes = in.read(tmp)) != -1;) { for (int readBytes = 0; (readBytes = in.read(tmp)) != -1; ) {
buf.write(tmp, 0, readBytes); buf.write(tmp, 0, readBytes);
} }
buf.flush(); buf.flush();
@ -73,7 +78,9 @@ public class SnappyLoaderTest
} }
@Test @Test
public void loadSnappyByDiffentClassloadersInTheSameJVM() throws Exception { public void loadSnappyByDiffentClassloadersInTheSameJVM()
throws Exception
{
// Parent class loader cannot see Snappy.class // Parent class loader cannot see Snappy.class
ClassLoader parent = this.getClass().getClassLoader().getParent(); ClassLoader parent = this.getClass().getClassLoader().getParent();
@ -89,17 +96,17 @@ public class SnappyLoaderTest
// Prepare the child class loaders which can load Snappy.class // Prepare the child class loaders which can load Snappy.class
URL classPath = new File("target/classes").toURI().toURL(); URL classPath = new File("target/classes").toURI().toURL();
ClassRealm L1 = cw.newRealm("l1", URLClassLoader.newInstance(new URL[] { classPath }, parent)); ClassRealm L1 = cw.newRealm("l1", URLClassLoader.newInstance(new URL[] {classPath}, parent));
ClassRealm L2 = cw.newRealm("l2", URLClassLoader.newInstance(new URL[] { classPath }, parent)); ClassRealm L2 = cw.newRealm("l2", URLClassLoader.newInstance(new URL[] {classPath}, parent));
// Actually load Snappy.class in a child class loader // Actually load Snappy.class in a child class loader
Class< ? > S1 = L1.loadClass("org.xerial.snappy.Snappy"); Class<?> S1 = L1.loadClass("org.xerial.snappy.Snappy");
Method m = S1.getMethod("compress", String.class); Method m = S1.getMethod("compress", String.class);
byte[] v = (byte[]) m.invoke(null, "hello world"); byte[] v = (byte[]) m.invoke(null, "hello world");
// Load Snappy.class from another child class loader // Load Snappy.class from another child class loader
Class< ? > S2 = L2.loadClass("org.xerial.snappy.Snappy"); Class<?> S2 = L2.loadClass("org.xerial.snappy.Snappy");
Method m2 = S2.getMethod("compress", String.class); Method m2 = S2.getMethod("compress", String.class);
byte[] v2 = (byte[]) m2.invoke(null, "hello world"); byte[] v2 = (byte[]) m2.invoke(null, "hello world");
@ -107,17 +114,22 @@ public class SnappyLoaderTest
} }
@Test @Test
public void load() throws Exception { public void load()
throws Exception
{
SnappyLoader.load(); SnappyLoader.load();
_logger.debug(Snappy.maxCompressedLength(1024)); _logger.debug(Snappy.maxCompressedLength(1024));
} }
@Test @Test
public void autoLoad() throws Exception { public void autoLoad()
throws Exception
{
_logger.debug(Snappy.maxCompressedLength(1024)); _logger.debug(Snappy.maxCompressedLength(1024));
} }
public static void main(String[] args) { public static void main(String[] args)
{
// Test for loading native library specified in -Djava.library.path // Test for loading native library specified in -Djava.library.path
System.setProperty(SnappyLoader.KEY_SNAPPY_USE_SYSTEMLIB, "true"); System.setProperty(SnappyLoader.KEY_SNAPPY_USE_SYSTEMLIB, "true");
_logger.debug(Snappy.maxCompressedLength(1024)); _logger.debug(Snappy.maxCompressedLength(1024));

163
src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java
View File

@ -29,8 +29,14 @@ import static org.junit.Assert.*;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.ByteOrder;
import org.junit.Test; import org.junit.Test;
import org.xerial.snappy.buffer.BufferAllocatorFactory;
import org.xerial.snappy.buffer.CachedBufferAllocator;
import org.xerial.snappy.buffer.DefaultBufferAllocator;
import org.xerial.util.FileResource; import org.xerial.util.FileResource;
import org.xerial.util.log.Logger; import org.xerial.util.log.Logger;
@ -39,7 +45,9 @@ public class SnappyOutputStreamTest
private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class); private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class);
@Test @Test
public void test() throws Exception { public void test()
throws Exception
{
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
SnappyOutputStream sout = new SnappyOutputStream(buf); SnappyOutputStream sout = new SnappyOutputStream(buf);
@ -49,7 +57,7 @@ public class SnappyOutputStreamTest
ByteArrayOutputStream orig = new ByteArrayOutputStream(); ByteArrayOutputStream orig = new ByteArrayOutputStream();
byte[] tmp = new byte[1024]; byte[] tmp = new byte[1024];
for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) { for (int readBytes = 0; (readBytes = input.read(tmp)) != -1; ) {
sout.write(tmp, 0, readBytes); sout.write(tmp, 0, readBytes);
orig.write(tmp, 0, readBytes); // preserve the original data orig.write(tmp, 0, readBytes); // preserve the original data
} }
@ -63,7 +71,7 @@ public class SnappyOutputStreamTest
ByteArrayOutputStream decompressed = new ByteArrayOutputStream(); ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
byte[] compressed = buf.toByteArray(); byte[] compressed = buf.toByteArray();
// decompress // decompress
for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length;) { for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length; ) {
int chunkSize = SnappyOutputStream.readInt(compressed, cursor); int chunkSize = SnappyOutputStream.readInt(compressed, cursor);
cursor += 4; cursor += 4;
byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)]; byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)];
@ -78,7 +86,9 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void bufferSize() throws Exception { public void bufferSize()
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b, 1500); SnappyOutputStream os = new SnappyOutputStream(b, 1500);
final int bytesToWrite = 5000; final int bytesToWrite = 5000;
@ -91,18 +101,21 @@ public class SnappyOutputStreamTest
os.close(); os.close();
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray())); SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
byte[] buf = new byte[bytesToWrite / 101]; byte[] buf = new byte[bytesToWrite / 101];
while (is.read(buf) != -1) {} while (is.read(buf) != -1) {
}
is.close(); is.close();
} }
@Test @Test
public void smallWrites() throws Exception { public void smallWrites()
throws Exception
{
byte[] orig = CalgaryTest.readFile("alice29.txt"); byte[] orig = CalgaryTest.readFile("alice29.txt");
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream out = new SnappyOutputStream(b); SnappyOutputStream out = new SnappyOutputStream(b);
for(byte c : orig) { for (byte c : orig) {
out.write(c); out.write(c);
} }
out.close(); out.close();
@ -111,7 +124,7 @@ public class SnappyOutputStreamTest
byte[] decompressed = new byte[orig.length]; byte[] decompressed = new byte[orig.length];
int cursor = 0; int cursor = 0;
int readLen = 0; int readLen = 0;
for(int i=0; i < decompressed.length && (readLen = is.read(decompressed, i, decompressed.length-i)) != -1; ) { for (int i = 0; i < decompressed.length && (readLen = is.read(decompressed, i, decompressed.length - i)) != -1; ) {
i += readLen; i += readLen;
} }
is.close(); is.close();
@ -120,11 +133,14 @@ public class SnappyOutputStreamTest
/** /**
* Compress the input array by passing it chunk-by-chunk to a SnappyOutputStream. * Compress the input array by passing it chunk-by-chunk to a SnappyOutputStream.
*
* @param orig the data to compress * @param orig the data to compress
* @param maxChunkSize the maximum chunk size, in bytes. * @param maxChunkSize the maximum chunk size, in bytes.
* @return the compressed bytes * @return the compressed bytes
*/ */
private static byte[] compressAsChunks(byte[] orig, int maxChunkSize) throws Exception { private static byte[] compressAsChunks(byte[] orig, int maxChunkSize)
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream out = new SnappyOutputStream(b); SnappyOutputStream out = new SnappyOutputStream(b);
@ -138,7 +154,9 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void batchingOfWritesShouldNotAffectCompressedDataSize() throws Exception { public void batchingOfWritesShouldNotAffectCompressedDataSize()
throws Exception
{
// Regression test for issue #100, a bug where the size of compressed data could be affected // Regression test for issue #100, a bug where the size of compressed data could be affected
// by the batching of writes to the SnappyOutputStream rather than the total amount of data // by the batching of writes to the SnappyOutputStream rather than the total amount of data
// written to the stream. // written to the stream.
@ -147,9 +165,12 @@ public class SnappyOutputStreamTest
byte[] expectedCompressedData = compressAsChunks(orig, Integer.MAX_VALUE); byte[] expectedCompressedData = compressAsChunks(orig, Integer.MAX_VALUE);
// Hardcoding an expected compressed size here will catch regressions that lower the // Hardcoding an expected compressed size here will catch regressions that lower the
// compression quality: // compression quality:
assertEquals(91013, expectedCompressedData.length); if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN)
assertEquals(90943, expectedCompressedData.length);
else
assertEquals(91013, expectedCompressedData.length);
// The chunk size should not affect the size of the compressed output: // The chunk size should not affect the size of the compressed output:
int[] chunkSizes = new int[] { 1, 100, 1023, 1024, 10000}; int[] chunkSizes = new int[] {1, 100, 1023, 1024, 10000};
for (int chunkSize : chunkSizes) { for (int chunkSize : chunkSizes) {
byte[] compressedData = compressAsChunks(orig, chunkSize); byte[] compressedData = compressAsChunks(orig, chunkSize);
assertEquals(String.format("when chunk size = %,d", chunkSize), expectedCompressedData.length, compressedData.length); assertEquals(String.format("when chunk size = %,d", chunkSize), expectedCompressedData.length, compressedData.length);
@ -158,7 +179,97 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void longArrayCompress() throws Exception { public void closeShouldBeIdempotent()
throws Exception
{
// Regression test for issue #107, a bug where close() was non-idempotent and would release
// its buffers to the allocator multiple times, which could cause scenarios where two open
// SnappyOutputStreams could share the same buffers, leading to stream corruption issues.
final BufferAllocatorFactory bufferAllocatorFactory = CachedBufferAllocator.getBufferAllocatorFactory();
final int BLOCK_SIZE = 4096;
// Create a stream, use it, then close it once:
ByteArrayOutputStream ba1 = new ByteArrayOutputStream();
SnappyOutputStream os1 = new SnappyOutputStream(ba1, BLOCK_SIZE, bufferAllocatorFactory);
os1.write(42);
os1.close();
// Create a new output stream, which should end up re-using the first stream's freed buffers
ByteArrayOutputStream ba2 = new ByteArrayOutputStream();
SnappyOutputStream os2 = new SnappyOutputStream(ba2, BLOCK_SIZE, bufferAllocatorFactory);
// Close the first stream a second time, which is supposed to be safe due to idempotency:
os1.close();
// Allocate a third output stream, which is supposed to get its own fresh set of buffers:
ByteArrayOutputStream ba3 = new ByteArrayOutputStream();
SnappyOutputStream os3 = new SnappyOutputStream(ba3, BLOCK_SIZE, bufferAllocatorFactory);
// Since the second and third streams should have distinct sets of buffers, writes to these
// streams should not interfere with one another:
os2.write(2);
os3.write(3);
os2.close();
os3.close();
SnappyInputStream in2 = new SnappyInputStream(new ByteArrayInputStream(ba2.toByteArray()));
assertEquals(2, in2.read());
in2.close();
SnappyInputStream in3 = new SnappyInputStream(new ByteArrayInputStream(ba3.toByteArray()));
assertEquals(3, in3.read());
in3.close();
}
@Test
public void writingToClosedStreamShouldThrowIOException()
throws IOException
{
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b);
os.close();
try {
os.write(4);
fail("Expected write() to throw IOException");
}
catch (IOException e) {
// Expected exception
}
try {
os.write(new int[] {1, 2, 3, 4});
fail("Expected write() to throw IOException");
}
catch (IOException e) {
// Expected exception
}
}
@Test
public void flushingClosedStreamShouldThrowIOException()
throws IOException
{
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b);
os.close();
try {
os.flush();
}
catch (IOException e) {
// Expected exception
}
}
@Test
public void closingStreamShouldMakeBuffersEligibleForGarbageCollection()
throws IOException
{
ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b, 4095, DefaultBufferAllocator.factory);
WeakReference<byte[]> inputBuffer = new WeakReference<byte[]>(os.inputBuffer);
WeakReference<byte[]> outputBuffer = new WeakReference<byte[]>(os.inputBuffer);
os.close();
System.gc();
assertNull(inputBuffer.get());
assertNull(outputBuffer.get());
}
@Test
public void longArrayCompress()
throws Exception
{
long[] l = new long[10]; long[] l = new long[10];
for (int i = 0; i < l.length; ++i) { for (int i = 0; i < l.length; ++i) {
l[i] = i % 3 + i * 11; l[i] = i % 3 + i * 11;
@ -176,15 +287,16 @@ public class SnappyOutputStreamTest
assertEquals(10 * 8, readBytes); assertEquals(10 * 8, readBytes);
assertArrayEquals(l, l2); assertArrayEquals(l, l2);
} }
@Test @Test
public void writeDoubleArray() throws Exception { public void writeDoubleArray()
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b); SnappyOutputStream os = new SnappyOutputStream(b);
double[] orig = new double[] { 1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20 }; double[] orig = new double[] {1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20};
os.write(orig); os.write(orig);
os.close(); os.close();
@ -197,11 +309,13 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void writeFloatArray() throws Exception { public void writeFloatArray()
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b); SnappyOutputStream os = new SnappyOutputStream(b);
float[] orig = new float[] { 1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f }; float[] orig = new float[] {1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f};
os.write(orig); os.write(orig);
os.close(); os.close();
@ -214,11 +328,13 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void writeIntArray() throws Exception { public void writeIntArray()
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b); SnappyOutputStream os = new SnappyOutputStream(b);
int[] orig = new int[] { 0, -1, -34, 43, 234, 34324, -234 }; int[] orig = new int[] {0, -1, -34, 43, 234, 34324, -234};
os.write(orig); os.write(orig);
os.close(); os.close();
@ -231,11 +347,13 @@ public class SnappyOutputStreamTest
} }
@Test @Test
public void writeShortArray() throws Exception { public void writeShortArray()
throws Exception
{
ByteArrayOutputStream b = new ByteArrayOutputStream(); ByteArrayOutputStream b = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(b); SnappyOutputStream os = new SnappyOutputStream(b);
short[] orig = new short[] { 0, -1, -34, 43, 234, 324, -234 }; short[] orig = new short[] {0, -1, -34, 43, 234, 324, -234};
os.write(orig); os.write(orig);
os.close(); os.close();
@ -246,5 +364,4 @@ public class SnappyOutputStreamTest
assertArrayEquals(orig, uncompressed); assertArrayEquals(orig, uncompressed);
} }
} }

88
src/test/java/org/xerial/snappy/SnappyTest.java
View File

@ -38,13 +38,17 @@ public class SnappyTest
private static Logger _logger = Logger.getLogger(SnappyTest.class); private static Logger _logger = Logger.getLogger(SnappyTest.class);
@Test @Test
public void getVersion() throws Exception { public void getVersion()
throws Exception
{
String version = Snappy.getNativeLibraryVersion(); String version = Snappy.getNativeLibraryVersion();
_logger.debug("version: " + version); _logger.debug("version: " + version);
} }
@Test @Test
public void directBufferCheck() throws Exception { public void directBufferCheck()
throws Exception
{
try { try {
ByteBuffer src = ByteBuffer.allocate(1024); ByteBuffer src = ByteBuffer.allocate(1024);
@ -59,11 +63,12 @@ public class SnappyTest
} }
fail("shouldn't reach here"); fail("shouldn't reach here");
} }
@Test @Test
public void directBuffer() throws Exception { public void directBuffer()
throws Exception
{
StringBuilder s = new StringBuilder(); StringBuilder s = new StringBuilder();
for (int i = 0; i < 20; ++i) { for (int i = 0; i < 20; ++i) {
@ -109,7 +114,9 @@ public class SnappyTest
} }
@Test @Test
public void bufferOffset() throws Exception { public void bufferOffset()
throws Exception
{
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA"; String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] orig = m.getBytes(); byte[] orig = m.getBytes();
@ -147,7 +154,9 @@ public class SnappyTest
} }
@Test @Test
public void byteArrayCompress() throws Exception { public void byteArrayCompress()
throws Exception
{
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA"; String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] input = m.getBytes(); byte[] input = m.getBytes();
@ -159,11 +168,12 @@ public class SnappyTest
int uncompressedSize = Snappy.uncompress(output, 0, compressedSize, uncompressed, 0); int uncompressedSize = Snappy.uncompress(output, 0, compressedSize, uncompressed, 0);
String m2 = new String(uncompressed); String m2 = new String(uncompressed);
assertEquals(m, m2); assertEquals(m, m2);
} }
@Test @Test
public void rangeCheck() throws Exception { public void rangeCheck()
throws Exception
{
String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA"; String m = "ACCAGGGGGGGGGGGGGGGGGGGGATAGATATTTCCCGAGATATTTTATATAAAAAAA";
byte[] input = m.getBytes(); byte[] input = m.getBytes();
byte[] output = new byte[Snappy.maxCompressedLength(input.length)]; byte[] output = new byte[Snappy.maxCompressedLength(input.length)];
@ -189,11 +199,12 @@ public class SnappyTest
bout.limit(bout.limit() - 1); bout.limit(bout.limit() - 1);
bout.position(1); bout.position(1);
assertFalse(Snappy.isValidCompressedBuffer(bout)); assertFalse(Snappy.isValidCompressedBuffer(bout));
} }
@Test @Test
public void highLevelAPI() throws Exception { public void highLevelAPI()
throws Exception
{
String m = "Hello! 01234 ACGDSFSDFJ World. FDSDF02394234 fdsfda03924"; String m = "Hello! 01234 ACGDSFSDFJ World. FDSDF02394234 fdsfda03924";
byte[] input = m.getBytes(); byte[] input = m.getBytes();
@ -205,7 +216,9 @@ public class SnappyTest
} }
@Test @Test
public void lowLevelAPI() throws Exception { public void lowLevelAPI()
throws Exception
{
String m = "Hello! 01234 ACGDSFSDFJ World. FDSDF02394234 fdsfda03924"; String m = "Hello! 01234 ACGDSFSDFJ World. FDSDF02394234 fdsfda03924";
byte[] input = m.getBytes(); byte[] input = m.getBytes();
@ -217,7 +230,9 @@ public class SnappyTest
} }
@Test @Test
public void simpleUsage() throws Exception { public void simpleUsage()
throws Exception
{
String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper" String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper"
+ " for using Snappy from Google (written in C++), a fast compresser/decompresser."; + " for using Snappy from Google (written in C++), a fast compresser/decompresser.";
@ -225,59 +240,72 @@ public class SnappyTest
byte[] uncompressed = Snappy.uncompress(compressed); byte[] uncompressed = Snappy.uncompress(compressed);
String result = new String(uncompressed, "UTF-8"); String result = new String(uncompressed, "UTF-8");
_logger.debug(result); _logger.debug(result);
} }
@Test @Test
public void floatArray() throws Exception { public void floatArray()
float[] data = new float[] { 1.0f, -0.3f, 1.3f, 234.4f, 34 }; throws Exception
{
float[] data = new float[] {1.0f, -0.3f, 1.3f, 234.4f, 34};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
float[] result = Snappy.uncompressFloatArray(compressed); float[] result = Snappy.uncompressFloatArray(compressed);
assertArrayEquals(data, result, 0.0f); assertArrayEquals(data, result, 0.0f);
} }
@Test @Test
public void doubleArray() throws Exception { public void doubleArray()
double[] data = new double[] { 1.0, -0.3, 1.3, 234.4, 34 }; throws Exception
{
double[] data = new double[] {1.0, -0.3, 1.3, 234.4, 34};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
double[] result = Snappy.uncompressDoubleArray(compressed); double[] result = Snappy.uncompressDoubleArray(compressed);
assertArrayEquals(data, result, 0.0f); assertArrayEquals(data, result, 0.0f);
} }
@Test @Test
public void longArray() throws Exception { public void longArray()
long[] data = new long[] { 2, 3, 15, 4234, 43251531412342342L, 23423422342L }; throws Exception
{
long[] data = new long[] {2, 3, 15, 4234, 43251531412342342L, 23423422342L};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
long[] result = Snappy.uncompressLongArray(compressed); long[] result = Snappy.uncompressLongArray(compressed);
assertArrayEquals(data, result); assertArrayEquals(data, result);
} }
@Test @Test
public void shortArray() throws Exception { public void shortArray()
short[] data = new short[] { 432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1 }; throws Exception
{
short[] data = new short[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
short[] result = Snappy.uncompressShortArray(compressed); short[] result = Snappy.uncompressShortArray(compressed);
assertArrayEquals(data, result); assertArrayEquals(data, result);
} }
@Test @Test
public void intArray() throws Exception { public void intArray()
int[] data = new int[] { 432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43 }; throws Exception
{
int[] data = new int[] {432, -32267, 1, 3, 34, 43, 34, Short.MAX_VALUE, -1, Integer.MAX_VALUE, 3424, 43};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
int[] result = Snappy.uncompressIntArray(compressed); int[] result = Snappy.uncompressIntArray(compressed);
assertArrayEquals(data, result); assertArrayEquals(data, result);
} }
@Test @Test
public void charArray() throws Exception { public void charArray()
char[] data = new char[] { 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!' }; throws Exception
{
char[] data = new char[] {'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!'};
byte[] compressed = Snappy.compress(data); byte[] compressed = Snappy.compress(data);
char[] result = Snappy.uncompressCharArray(compressed); char[] result = Snappy.uncompressCharArray(compressed);
assertArrayEquals(data, result); assertArrayEquals(data, result);
} }
@Test @Test
public void string() throws Exception { public void string()
throws Exception
{
String s = "Hello Snappy! Snappy! Snappy!"; String s = "Hello Snappy! Snappy! Snappy!";
byte[] compressed = Snappy.compress(s); byte[] compressed = Snappy.compress(s);
String uncompressedString = Snappy.uncompressString(compressed); String uncompressedString = Snappy.uncompressString(compressed);
@ -285,9 +313,11 @@ public class SnappyTest
} }
@Test @Test
public void isValidCompressedData() throws Exception { public void isValidCompressedData()
throws Exception
{
byte[] b = new byte[] { (byte) 91, (byte) 34, (byte) 80, (byte) 73, (byte) 34, (byte) 93 }; byte[] b = new byte[] {(byte) 91, (byte) 34, (byte) 80, (byte) 73, (byte) 34, (byte) 93};
assertFalse(Snappy.isValidCompressedBuffer(b)); assertFalse(Snappy.isValidCompressedBuffer(b));
@ -298,7 +328,5 @@ public class SnappyTest
catch (IOException e) { catch (IOException e) {
_logger.debug(e); _logger.debug(e);
} }
} }
} }

2
version.sbt
View File

@ -1 +1 @@
version in ThisBuild := "1.1.2-SNAPSHOT" version in ThisBuild := "1.1.3-SNAPSHOT"