Compare commits

..

No commits in common. "master" and "1.1.1.5" have entirely different histories.

139 changed files with 5750 additions and 13095 deletions

View File

@ -1,2 +0,0 @@
# Scala Steward: Reformat with scalafmt 3.7.2
2b37d1ce0bbed3d8d03a66292e7d90dcde805624

2
.gitattributes vendored
View File

@ -1,2 +0,0 @@
sbt text eol=lf

View File

@ -1,9 +0,0 @@
# Set update schedule for GitHub Actions
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
# Check for updates to GitHub Actions every week
interval: "weekly"

View File

@ -1,55 +0,0 @@
name-template: 'v$RESOLVED_VERSION'
tag-template: 'v$RESOLVED_VERSION'
categories:
- title: '🔥 Breaking Changes'
labels:
- 'breaking'
- title: '🚀 Features'
labels:
- 'feature'
- 'enhancement'
- title: '🐛 Bug Fixes'
labels:
- 'bug'
- title: '👋 Deprecated'
labels:
- 'deprecation'
- title: '🔗 Dependency Updates'
labels:
- 'library-update'
- 'dependencies'
- title: '🛠 Internal Updates'
labels:
- 'internal'
- 'kaizen'
- 'test-library-update'
- 'sbt-plugin-update'
- title: '📚 Docs'
labels:
- 'doc'
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
template: |
## What's Changed
$CHANGES
**Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION
autolabeler:
- label: 'doc'
files:
- '*.md'
- label: 'feature'
title:
- '/(support|feature)/i'
- label: 'bug'
title:
- '/fix/i'
- label: 'internal'
title:
- '/internal/i'
- label: 'deprecation'
title:
- '/deprecate/i'

31
.github/release.yml vendored
View File

@ -1,31 +0,0 @@
changelog:
categories:
- title: '🔥 Breaking Changes'
labels:
- 'breaking'
- title: '👋 Deprecated'
labels:
- 'deprecation'
- title: '🚀 Features'
labels:
- 'feature'
- 'enhancement'
- title: '🐛 Bug Fixes'
labels:
- 'bug'
- title: '🔗 Dependency Updates'
labels:
- 'library-update'
- 'dependencies'
- title: '🛠 Internal Updates'
labels:
- 'internal'
- 'kaizen'
- 'test-library-update'
- 'sbt-plugin-update'
- title: '📚 Docs'
labels:
- 'doc'
- title: Other Changes
labels:
- "*"

View File

@ -1,48 +0,0 @@
name: Build Native
on:
workflow_dispatch:
push:
branches:
- master
- main
paths:
- 'src/main/resources/org/xerial/snappy/VERSION'
- 'Makefile'
- 'Makefile.common'
- '**/*.h'
- '**/*.cpp'
- .github/workflows/build-native.yml
pull_request:
paths:
- 'src/main/resources/org/xerial/snappy/VERSION'
- 'Makefile'
- 'Makefile.common'
- '**/*.h'
- '**/*.cpp'
jobs:
build:
permissions:
contents: write
pull-requests: write
name: Build native libraries
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2.0
with:
cmake-version: '3.16.x'
- name: Build native libraries
run: make clean-native native-all
env:
OCI_EXE: docker
- name: Create Pull Request
if: ${{ github.event_name != 'pull_request' }}
uses: peter-evans/create-pull-request@v7
with:
title: Update native libraries
commit-message: Update native libraries for ${{ github.sha }}
branch: update-native-libs
labels: library-update

View File

@ -1,27 +0,0 @@
name: Release Drafter
on:
# pull_request event is required only for autolabeler
pull_request:
# Only following types are handled by the action, but one can default to all as well
types: [opened, reopened, synchronize]
# pull_request_target event is required for autolabeler to support PRs from forks
pull_request_target:
types: [opened, reopened, synchronize]
permissions:
contents: read
jobs:
update_PR_labels:
permissions:
contents: read
# write permission is required for autolabeler
# otherwise, read permission is required at least
pull-requests: write
runs-on: ubuntu-latest
steps:
# Runs only pull-request labeler
- uses: release-drafter/release-drafter@v6
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,18 +0,0 @@
name: Release Note
on:
push:
tags:
- v*
workflow_dispatch:
jobs:
release:
name: Create a new release note
runs-on: ubuntu-latest
steps:
- name: Create a release note
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release create "$GITHUB_REF_NAME" --repo="$GITHUB_REPOSITORY" --generate-notes

View File

@ -1,37 +0,0 @@
name: Release
on:
push:
tags:
- v*
workflow_dispatch:
jobs:
publish_jvm:
name: Publish
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 10000
# Fetch all tags so that sbt-dynver can find the previous release version
- run: git fetch --tags -f
# Install OpenJDK 11
- uses: actions/setup-java@v4
with:
distribution: 'zulu'
java-version: '11'
- name: Setup GPG
env:
PGP_SECRET: ${{ secrets.PGP_SECRET }}
run: echo $PGP_SECRET | base64 --decode | gpg --import --batch --yes
- name: Build bundle
env:
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
run: |
./sbt publishSigned
- name: Release to Sonatype
env:
SONATYPE_USERNAME: '${{ secrets.SONATYPE_USER }}'
SONATYPE_PASSWORD: '${{ secrets.SONATYPE_PASS }}'
run: ./sbt sonatypeBundleRelease

View File

@ -1,39 +0,0 @@
name: Snapshot Release
on:
push:
branches:
- master
paths:
- '**.scala'
- '**.java'
- '**.sbt'
- 'src/main/resources/org/xerial/snappy/native/**'
- '.github/workflows/snapshot.yml'
tag:
- '!*'
workflow_dispatch:
jobs:
publish_snapshots:
name: Publish snapshots
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 10000
# Fetch all tags so that sbt-dynver can find the previous release version
- run: git fetch --tags
- uses: olafurpg/setup-scala@v14
with:
java-version: adopt@1.11
- uses: actions/cache@v4
with:
path: ~/.cache
key: ${{ runner.os }}-snapshot-${{ hashFiles('**/*.sbt') }}
restore-keys: ${{ runner.os }}-snapshot-
- name: Publish snapshots
env:
SONATYPE_USERNAME: '${{ secrets.SONATYPE_USER }}'
SONATYPE_PASSWORD: '${{ secrets.SONATYPE_PASS }}'
run: ./sbt publish

View File

@ -1,52 +0,0 @@
name: CI
on:
pull_request:
paths:
- '**.scala'
- '**.java'
- '**.sbt'
- '.github/workflows/*.yml'
- '**.so'
- '**.dll'
- 'src/main/resources/**'
- 'project/build.properties'
push:
branches:
- master
- main
paths:
- '**.scala'
- '**.java'
- '**.sbt'
- '.github/workflows/*.yml'
- 'src/main/resources/org/xerial/snappy/**'
- 'project/build.properties'
jobs:
code_format:
name: code format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: scalafmt test
run: ./sbt scalafmtCheckAll
test-jdk:
strategy:
matrix:
version: [ '8', '11', '17', '21' ]
name: test jdk${{ matrix.version }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: 'zulu'
java-version: ${{ matrix.version }}
- uses: actions/cache@v4
with:
path: ~/.cache
key: ${{ runner.os }}-jdk${{ matrix.version }}-${{ hashFiles('**/*.sbt') }}
restore-keys: ${{ runner.os }}-jdk${{ matrix.version }}-
- name: Test
run: ./sbt test

5
.gitignore vendored
View File

@ -17,14 +17,9 @@ lib_managed/
src_managed/
project/boot/
project/plugins/project/
.bsp
# Scala-IDE specific
.scala_dependencies
atlassian-ide-plugin.xml
src/main/java/**/*.h
.metals
.vscode
.bloop

View File

@ -1,5 +0,0 @@
version = 3.8.6
runner.dialect = scala213source3
maxColumn = 180
style = defaultWithAlign
optIn.breaksInsideChains = true

View File

@ -0,0 +1,6 @@
#Tue Mar 29 16:49:37 JST 2011
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.6

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,9 @@
#Tue Mar 29 16:49:37 JST 2011
activeProfiles=
eclipse.preferences.version=1
fullBuildGoals=process-test-resources
includeModules=false
resolveWorkspaceProjects=true
resourceFilterGoals=process-resources resources\:testResources
skipCompilerPlugin=true
version=1

View File

@ -1,19 +1,15 @@
language: scala
scala:
- 2.11.1
jdk:
- openjdk6
- openjdk7
- oraclejdk7
branches:
only:
- master
- master
- develop
language: java
script: ./sbt test
jdk:
- openjdk11
os: linux
arch:
- s390x
- ppc64le
script:
# - sudo apt-get install -y openjdk-11-jdk;
# - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-s390x && export PATH=$JAVA_HOME/bin:$PATH;
- ./sbt test

119
BUILD.md
View File

@ -1,119 +0,0 @@
# Notes for building snappy-java
snappy-java supports Windows, Mac OS X, Linux (x86, x86_64, arm, etc...). If your platform is not supported, you need to build native libraries by yourself.
## Requited Tools
- Java 7 or higher
- Maven3 (mvn)
- GNU make, autotools
## Building snappy-java
To build jar file of snappy-java, type:
```
$ make
```
A native library for your machine environment and a jar package target/snappy-java-(version).jar are produced in the target folder.
### Rebuild the native library for your platform
```
$ make clean-native native
```
## Platform specific tips
After snappy-java 1.1.3, we are using docker images of cross compilers. So no longer need to build native libraries by actually running the target OS.
The following notes are obsolete, but preserved here for future references.
### Windows (32/64-bit)
* GNU make
* And also tar, curl, cp, rm, grep commands are needed. (I use Cygwin and MinGW for building snappy-java in Windows)
### Windows (32-bit only)
* Install MinGW http://www.mingw.org/
* Set PATH to the following command in MinGW package
- mingw32-g++
- strip
To build x86 (32bit) dll under 64-bit Windows, use "make win32" target.
### Windows (64-bit only)
* Download MinGW-w64 http://sourceforge.net/projects/mingw-w64/
* Set PATH to the following commands in the downloaded archive:
- x86_64-w64-mingw32-g++
- x86_64-w64-mingw32-strip
NOTICE: Do not use the Cygwin version of MinGW-w64. It fails to build assemblies for 64bit environment.
### Linux (32/64-bit)
* gcc-4.5.x or higher is necessary because snappy-java uses -static-libstdc++ option. It is possible to use gcc-4.3.x but a dependency to libstdc++ remains in the generated jar file; That means if another version of libstdc++ is used, snappy-java might not work correctly.
* You can build 32-bit native library with 64-bit Linux machine (do make linux32)
### Mac
* Install gcc, make, etc. included in Mac OS X install disk. (X Code). And also intall libtool:
```
$ brew install libtool
```
## Building Linux x86\_64 binary
(obsolete: snappy-java now uses a docker image `xerial/centos5-linux-x86_86-pic` which contains g++ built with `-fPIC` option. )
snappy-java tries to static link libstdc++ to increase the availability for various Linux versions. However, standard distributions of 64-bit Linux OS rarely provide libstdc++ compiled with `-fPIC` option. I currently uses custom g++, compiled as follows:
```
$ cd work
$ wget (gcc-4.8.3 source)
$ tar xvfz (gcc-4.8.3.tar.gz)
$ cd gcc-4.8.3
$ ./contrib/download_prerequisites
$ cd ..
$ mkdir objdir
$ cd objdir
$ ../gcc-4.8.3/configure --prefix=$HOME/local/gcc-4.8.3 CXXFLAGS=-fPIC CFLAGS=-fPIC --enable-languages=c,c++
$ make
$ make install
```
This g++ build enables static linking of libstdc++. For more infomation on building GCC, see GCC's home page.
## Building Linux s390/s390x binaries
Older snapshots of snappy contain a buggy config.h.in that does not work properly on some big-endian platforms like Linux on IBM z (s390/s390x). Building snappy-java on s390/s390x requires fetching the snappy source from GitHub, and processing the source with autoconf to obtain a usable config.h. On a RHEL s390x system, these steps produced a working 64-bit snappy-java build (the process should be similar for other distributions):
$ sudo yum install java-1.7.1-ibm-devel libstdc++-static-devel
$ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x
$ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1
## Activating SSE2/AVX2 instructions in BitShuffle
The most of the native libraries that snappy-java contains disable SSE2/AVX2 instructions in terms of portability (SSE2 is enabled only in Linux/x86_64 platforms). To enable AVX2 instructions, you need to compile as follows:
$ make CXXFLAGS_BITSHUFFLE=-mavx2 # -msse2 for SSE2 instructions
## Cross-compiling for other platforms (obsolete)
The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM:
$ make linux32 win32 win64 linux-arm linux-armhf linux-aarch64
If you append `snappy` to the line above, it will also build the native library for the current platform and then build the snappy-java JAR (containing all native libraries built so far).
Of course, you must first have the necessary cross-compilers and development libraries installed for each target CPU and OS. For example, on Ubuntu 12.04 for x86-64, install the following packages for each target:
* linux32: `sudo apt-get install g++-multilib libc6-dev-i386 lib32stdc++6`
* win32: `sudo apt-get install g++-mingw-w64-i686`
* win64: `sudo apt-get install g++-mingw-w64-x86-64`
* arm: `sudo apt-get install g++-arm-linux-gnueabi`
* armhf: `sudo apt-get install g++-arm-linux-gnueabihf`
* aarch64: `sudo apt-get install g++-aarch64-linux`
Unfortunately, cross-compiling for Mac OS X is not currently possible; you must compile within OS X.
If you are using Mac and openjdk7 (or higher), use the following option:
$ make native LIBNAME=libsnappyjava.dylib

78
INSTALL Executable file
View File

@ -0,0 +1,78 @@
[Installation note of snappy-java]
If your OS platform is one of the Windows/Linux/Mac OS X (32/64 bit Intel CPUs), the installation process described here is unnecessary. Simply put snappy-java-(version).jar to your Java classpath. If your OS is not listed above, create your own snappy-java-(version).jar file as follows.
= Required tools for building snappy-java =
[For all platforms]
* Java 6 (JDK1.6) http://java.sun.com/
- set JAVA_HOME environment variable to the Java installation folder (e.g. JAVA_HOME=C:/Program Files/Java/jdk1.6.0_24 in Windows)
* Maven 3.x http://maven.apache.org/
- Check mvn command can be used from your command line.
[Windows (32/64-bit)]
* GNU make
* And also tar, curl, cp, rm, grep commands are needed. (I use Cygwin and MinGW for building snappy-java in Windows)
[Windows (32-bit only)]
* Install MinGW http://www.mingw.org/
* Set PATH to the following command in MinGW package
- mingw32-g++
- strip
To build x86 (32bit) dll under 64-bit Windows, use "make win32" target.
[Windows (64-bit only)]
* Download MinGW-w64 http://sourceforge.net/projects/mingw-w64/
* Set PATH to the following commands in the downloaded archive:
- x86_64-w64-mingw32-g++
- x86_64-w64-mingw32-strip
NOTICE: Do not use the Cygwin version of MinGW-w64. It fails to build assemblies for 64bit environment.
[Linux (32/64-bit)]
* gcc-4.5.x or higher is necessary because snappy-java uses -static-libstdc++ option. It is possible to use gcc-4.3.x but a dependency to libstdc++ remains in the generated jar file; That means if another version of libstdc++ is used, snappy-java might not work correctly.
* You can build 32-bit native library with 64-bit Linux machine (do make linux32)
[Mac]
* Install gcc, make, etc. included in Mac OS X install disk. (X Code)
* Install mercurial using Mac Ports http://www.macports.org/
= Building snappy-java =
To build jar file of snappy-java, type:
$ make
A native library for your machine environment and a jar package target/snappy-java-(version).jar are produced in the target folder.
= Building only the native library =
$ make native
= Rebuild the native library for your platform =
$ make clean-native native
= Using system installed libsnappyjava (or snappyjava.dll) =
Set org.xerial.snappy.use.systemlib system property to true:
java -Djava.library.path=(path to the installed snappyjava lib) -Dorg.xerial.snappy.use.systemlib=true ...
With this setting snappy-java does not use bundled native libraries. Insted it tries to load native library installed at the path specified in java.library.path.
= Configure snappy-java using property file =
Prepare org-xerial-snappy.properties file (under the root path of your library) in Java's property file format.
Here is a list of the available properties:
* org.xerial.snappy.lib.path (directory containing a snappyjava's native library)
* org.xerial.snappy.lib.name (library file name)
* org.xerial.snappy.tempdir (temporary directory to extract a native library bundled in snappy-java)
* org.xerial.snappy.use.systemlib (if this value is true, use system installed libsnappyjava.so looking the path specified by java.library.path)

248
Makefile
View File

@ -1,138 +1,83 @@
include Makefile.common
$(info OS_NAME:$(OS_NAME), OS_ARCH:$(OS_ARCH))
MVN:=mvn
SBT:=./sbt
all: snappy
SNAPPY_OUT:=$(TARGET)/snappy-$(SNAPPY_VERSION)-$(os_arch)
SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz
SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc snappy.cc
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
SNAPPY_OUT:=$(TARGET)/$(snappy)-$(os_arch)
SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(VERSION).tar.gz
SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(VERSION)
SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
# Change this variable to use a specific git commit
SNAPPY_GIT_REV:=$(SNAPPY_VERSION)
SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) SnappyNative.o)
SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
SNAPPY_CMAKE_CACHE=$(SNAPPY_OUT)/CMakeCache.txt
BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz
BITSHUFFLE_C:=bitshuffle_core.c iochain.c
BITSHUFFLE_SRC_DIR:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION)/src
BITSHUFFLE_SRC:=$(addprefix $(BITSHUFFLE_SRC_DIR)/,$(BITSHUFFLE_C))
BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log
$(BITSHUFFLE_ARCHIVE):
@mkdir -p $(@D)
curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz
$(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET)
touch $@
$(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED)
$(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) $(CXXFLAGS_BITSHUFFLE) -c $< -o $@
SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o)
CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(SNAPPY_OUT) -I$(BITSHUFFLE_SRC_DIR)
ifndef CXXFLAGS_BITSHUFFLE
ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64)
# SSE2 is supported in all the x86_64 platforms and AVX2 is only supported
# in the small part of them. gcc in linux/x86_64 typically enables SSE2 by default though,
# we explicitly set flags below to make this precondition clearer.
CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -msse2
else
# Undefined macros to generate a platform-independent binary
CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -U__SSE2__
ifdef USE_GIT
ifndef GIT_REPO_URL
$(warning GIT_REPO_URL is not set when using git)
endif
ifndef GIT_SNAPPY_BRANCH
$(warning GIT_SNAPPY_BRANCH is not set when using git)
endif
endif
CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR)
ifeq ($(OS_NAME),SunOS)
TAR:= gtar
else
ifeq ($(OS_NAME),AIX)
TAR:= gtar
else
TAR:= tar
endif
endif
$(SNAPPY_ARCHIVE):
@mkdir -p $(@D)
curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz
curl -o$@ http://snappy.googlecode.com/files/snappy-$(VERSION).tar.gz
$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET)
$(TAR) xvfz $< -C $(TARGET)
touch $@
cd $(SNAPPY_SRC_DIR) && ./configure
$(SNAPPY_GIT_UNPACKED):
@mkdir -p $(SNAPPY_OUT)
rm -rf $(SNAPPY_SRC_DIR)
@mkdir -p $(SNAPPY_SRC_DIR)
git clone $(SNAPPY_GIT_REPO_URL) $(SNAPPY_SRC_DIR)
git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(SNAPPY_VERSION) $(SNAPPY_GIT_REV)
cd $(SNAPPY_SRC_DIR) && git submodule update --init
git clone $(GIT_REPO_URL) $(SNAPPY_SRC_DIR)
git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(GIT_SNAPPY_BRANCH) $(GIT_SNAPPY_BRANCH)
touch $@
cd $(SNAPPY_SRC_DIR) && ./configure
$(SNAPPY_CMAKE_CACHE): $(SNAPPY_GIT_UNPACKED)
@mkdir -p $(SNAPPY_OUT)
cd $(SNAPPY_OUT) && cmake $(SNAPPY_CMAKE_OPTS) ../../$(SNAPPY_SRC_DIR)
touch $@
jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h
jni-header: $(SNAPPY_GIT_UNPACKED) $(BITSHUFFLE_UNPACKED) $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
snappy-header: $(SNAPPY_CMAKE_CACHE)
$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java
$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/snappy/SnappyNative.java
@mkdir -p $(TARGET)/jni-classes
$(JAVAC) -source 1.7 -target 1.7 -h $(SRC)/org/xerial/snappy/ -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class
$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative
$(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java
@mkdir -p $(TARGET)/jni-classes
$(JAVAC) -source 1.7 -target 1.7 -h $(SRC)/org/xerial/snappy/ -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
$(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class
$(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
# aarch64 can use big-endian optimzied code
ifeq ($(OS_ARCH),aarch64)
ifeq ($(ENDIANESS),$(BIG_ENDIAN))
SNAPPY_CXX_OPTS:=-DSNAPPY_IS_BIG_ENDIAN
endif
ifndef USE_GIT
$(SNAPPY_SRC): $(SNAPPY_UNPACKED)
else
$(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
endif
$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc
$(SNAPPY_OUT)/%.o : $(SNAPPY_SRC_DIR)/%.cc
@mkdir -p $(@D)
$(CXX) $(SNAPPY_CXX_OPTS) $(CXXFLAGS) -c $< -o $@
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
@mkdir -p $(@D)
$(CXX) $(SNAPPY_CXX_OPTS) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h
$(SNAPPY_OUT)/SnappyNative.o : $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ)
$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
# Workaround for strip Protocol error when using VirtualBox on Mac
cp $@ /tmp/$(@F)
$(STRIP) /tmp/$(@F)
cp /tmp/$(@F) $@
clean-native:
$(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ)
$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
$(STRIP) $@
clean-native:
rm -rf $(SNAPPY_OUT)
clean:
@ -142,106 +87,79 @@ NATIVE_DIR:=src/main/resources/org/xerial/snappy/native/$(OS_NAME)/$(OS_ARCH)
NATIVE_TARGET_DIR:=$(TARGET)/classes/org/xerial/snappy/native/$(OS_NAME)/$(OS_ARCH)
NATIVE_DLL:=$(NATIVE_DIR)/$(LIBNAME)
snappy-jar-version:=snappy-java-$(shell ./script/dynver.sh | cut -d'=' -f2 | sed 's/[ \"]//g')
snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s+\"(.*)\"/\1/" version.sbt | sed -e "/^$$/d")
jar-version:
echo $(snappy-jar-version)
native: jni-header snappy-header $(NATIVE_DLL)
native-nocmake: jni-header $(NATIVE_DLL)
ifndef USE_GIT
native: $(SNAPPY_UNPACKED) $(NATIVE_DLL)
else
native: $(SNAPPY_GIT_UNPACKED) $(NATIVE_DLL)
endif
snappy: native $(TARGET)/$(snappy-jar-version).jar
native-all: native native-arm clean-docker mac64 win32 win64 linux32 linux64 linux-ppc64le linux-riscv64 linux-s390x
ifdef CI
# Clean docker images within CI to avoid no space left error
DOCKER_POST_PROCESS:=docker system prune --all --force --volumes
else
DOCKER_POST_PROCESS:=
endif
clean-docker:
$(DOCKER_POST_PROCESS)
$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
@mkdir -p $(@D)
cp $(SNAPPY_OUT)/$(LIBNAME) $@
cp $< $@
@mkdir -p $(NATIVE_TARGET_DIR)
cp $(SNAPPY_OUT)/$(LIBNAME) $(NATIVE_TARGET_DIR)/$(LIBNAME)
cp $< $(NATIVE_TARGET_DIR)/$(LIBNAME)
package: $(TARGET)/$(snappy-jar-version).jar
$(TARGET)/$(snappy-jar-version).jar:
$(SBT) package
$(TARGET)/$(snappy-jar-version).jar:
$(SBT) package
test: $(NATIVE_DLL)
$(SBT) test
DOCKER_RUN_OPTS:=--rm
win32:
$(MAKE) native CROSS_PREFIX=i686-w64-mingw32- OS_NAME=Windows OS_ARCH=x86
# for cross-compilation on Ubuntu, install the g++-mingw-w64-x86-64 package
win64:
$(MAKE) native CROSS_PREFIX=x86_64-w64-mingw32- OS_NAME=Windows OS_ARCH=x86_64
win32: jni-header
./docker/dockcross-windows-x86 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native snappy-header native CROSS_PREFIX=i686-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86 SNAPPY_CMAKE_OPTS="-DHAVE_SYS_UIO_H=0"'
win64: jni-header
./docker/dockcross-windows-x64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native snappy-header native CROSS_PREFIX=x86_64-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86_64 SNAPPY_CMAKE_OPTS="-DHAVE_SYS_UIO_H=0"'
# deprecated
mac32: jni-header
mac32:
$(MAKE) native OS_NAME=Mac OS_ARCH=x86
mac64: jni-header
docker run -i $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin multiarch/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64
linux32: jni-header
docker run $(DOCKER_RUN_OPTS) -i -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86'
linux64: jni-header
docker run $(DOCKER_RUN_OPTS) -i -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86_64'
linux32:
$(MAKE) native OS_NAME=Linux OS_ARCH=x86
freebsd64:
$(MAKE) native OS_NAME=FreeBSD OS_ARCH=x86_64
# For ARM
native-arm: linux-arm64 linux-android-arm linux-android-aarch64 linux-arm linux-armv6 linux-armv7 windows-aarch64
# for cross-compilation on Ubuntu, install the g++-arm-linux-gnueabi package
linux-arm:
$(MAKE) native CROSS_PREFIX=arm-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm
# for cross-compilation on Ubuntu, install the g++-arm-linux-gnueabihf package
linux-armhf:
$(MAKE) native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armhf
# TODO: CROSS_PREFIX can be replaced with ${CROSS_ROOT}/bin/${CROSS_TRIPLE}- in Makefile.common
linux-arm: jni-header
./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/armv5-unknown-linux-gnueabi/bin//armv5-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm'
clean-native-linux32:
$(MAKE) clean-native OS_NAME=Linux OS_ARCH=x86
linux-armv6: jni-header
./docker/dockcross-armv6 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=armv6'
linux-armv7: jni-header
./docker/dockcross-armv7 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=armv7'
linux-android-arm: jni-header
./docker/dockcross-android-arm -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/arm-linux-androideabi/bin/arm-linux-androideabi- OS_NAME=Linux OS_ARCH=android-arm'
linux-android-aarch64: jni-header
./docker/dockcross-android-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=android-aarch64'
linux-ppc64le: jni-header
./docker/dockcross-ppc64le -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64le-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64le'
linux-ppc64: jni-header
./docker/dockcross-ppc64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64'
linux-arm64: jni-header
./docker/dockcross-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=aarch64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64'
linux-riscv64: jni-header
./docker/dockcross-riscv64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/riscv64-unknown-linux-gnu/bin/riscv64-unknown-linux-gnu- OS_NAME=Linux OS_ARCH=riscv64'
linux-s390x: jni-header
./docker/dockcross-s390x -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/s390x-ibm-linux-gnu/bin/s390x-ibm-linux-gnu- OS_NAME=Linux OS_ARCH=s390x'
windows-aarch64: jni-header
./docker/dockcross-windows-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/aarch64-w64-mingw32-cross/bin/aarch64-w64-mingw32- OS_NAME=Windows OS_ARCH=aarch64'
clean-native-win32:
$(MAKE) clean-native OS_NAME=Windows OS_ARCH=x86
javadoc:
$(SBT) doc
install-m2:
$(SBT) publishM2
googlecode-upload: googlecode-lib-upload googlecode-src-upload
googlecode-lib-upload: $(TARGET)/snappy-java-$(VERSION)-lib.upload
googlecode-src-upload: $(TARGET)/snappy-java-$(VERSION)-src.upload
GOOGLECODE_USER:=leo@xerial.org
$(TARGET)/snappy-java-$(VERSION)-lib.upload:
./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).jar
touch $@
$(TARGET)/snappy-java-$(VERSION)-src.upload:
./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).tar.gz
touch $@

View File

@ -1,22 +1,20 @@
TARGET:=target
SRC:=src/main/java
include src/main/resources/org/xerial/snappy/VERSION
include $(SRC)/org/xerial/snappy/VERSION
ifndef JAVA_HOME
$(warning Set JAVA_HOME environment variable)
$(error Set JAVA_HOME environment variable)
endif
JAVA := "$$JAVA_HOME/bin/java"
JAVAC := "$$JAVA_HOME/bin/javac"
JAVAC := "$$JAVA_HOME/bin/javac"
JAVAH := "$$JAVA_HOME/bin/javah"
LITTLE_ENDIAN := "Little Endian"
BIG_ENDIAN := "Big Endian"
OSINFO_CLASS := org.xerial.snappy.OSInfo
OSINFO_PROG := lib/org/xerial/snappy/OSInfo.class
## building OSInfo.java
## building OSInfo.java
#$(info compiling OSInfo.java)
#$(shell mkdir -p lib)
#$(shell $(JAVAC) src/main/java/org/xerial/snappy/OSInfo.java -d lib)
@ -26,12 +24,6 @@ OS_ARCH := $(shell $(JAVA) -cp lib $(OSINFO_CLASS) --arch)
LIB_FOLDER := $(shell $(JAVA) -cp lib $(OSINFO_CLASS))
IBM_JDK_LIB := lib/inc_ibm
ifeq ($(OS_NAME),Mac)
ENDIANESS:="Little Endian"
else
ENDIANESS:=$(shell lscpu | grep "Byte Order" |cut -d ":" -f2)
endif
# Windows uses different path separators
ifeq ($(OS_NAME),Windows)
sep := ;
@ -39,7 +31,7 @@ else
sep := :
endif
snappy := snappy-$(SNAPPY_VERSION)
snappy := snappy-$(VERSION)
jni_md := $(shell find -L "$(JAVA_HOME)" -name jni_md.h | head -1)
@ -50,8 +42,7 @@ endif
# os=Default is meant to be generic unix/linux
# To support new CPU architecture, add a new target name here
known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-android-aarch64 Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 Mac-aarch64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 Windows-aarch64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64 Linux-riscv Linux-riscv64 Linux-loongarch Linux-loongarch64
known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armhf Linux-ppc64 Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc64
os_arch := $(OS_NAME)-$(OS_ARCH)
IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7"))
@ -76,124 +67,68 @@ CROSS_PREFIX :=
Default_CXX := $(CROSS_PREFIX)g++
Default_STRIP := $(CROSS_PREFIX)strip
Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11
Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
Default_LINKFLAGS := -shared -static
Default_LIBNAME := libsnappyjava.so
Default_SNAPPY_FLAGS :=
Default_SNAPPY_FLAGS :=
Linux-x86_CXX := $(CROSS_PREFIX)g++
Linux-x86_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-x86_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11
Linux-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32
else
Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11
Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32
endif
Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_LIBNAME := libsnappyjava.so
Linux-x86_SNAPPY_FLAGS:=
Linux-x86_SNAPPY_FLAGS:=
Linux-x86_64_CXX := $(CROSS_PREFIX)g++
Linux-x86_64_CXX := $(CROSS_PREFIX)g++
Linux-x86_64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
endif
Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_64_LIBNAME := libsnappyjava.so
Linux-x86_64_SNAPPY_FLAGS :=
Linux-x86_64_SNAPPY_FLAGS :=
Linux-ppc_CXX := g++
Linux-ppc_STRIP := strip
Linux-ppc64_CXX := g++
Linux-ppc64_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32 -std=c++11
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32 -std=c++11
endif
Linux-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc_LIBNAME := libsnappyjava.so
Linux-ppc_SNAPPY_FLAGS :=
Linux-ppc64le_CXX := $(CROSS_PREFIX)g++
Linux-ppc64le_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11
endif
# ppcle64 GLIBC is at 2.17; so disable __tls_get_addr_opt which is dependent on 2.22;
Linux-ppc64le_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -Wl,--no-tls-optimize,--no-tls-get-addr-optimize
Linux-ppc64le_LIBNAME := libsnappyjava.so
Linux-ppc64le_SNAPPY_FLAGS :=
Linux-ppc64_CXX := $(CROSS_PREFIX)g++
Linux-ppc64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11
Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC
endif
Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64_LIBNAME := libsnappyjava.so
Linux-ppc64_SNAPPY_FLAGS :=
AIX-ppc_CXX := g++
AIX-ppc_STRIP := strip
AIX-ppc_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32 -std=c++11
else
AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32 -std=c++11
endif
AIX-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc_SNAPPY_FLAGS :=
AIX-ppc64_CXX := g++
AIX-ppc64_STRIP := strip -X64
AIX-ppc64_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64 -std=c++11
AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64 -std=c++11
AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64
endif
AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc64_SNAPPY_FLAGS :=
Linux-s390_CXX := g++
Linux-s390_STRIP := strip
ifeq ($(IBM_JDK_7),)
Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31 -std=c++11
else
Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31 -std=c++11
endif
Linux-s390_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390_LIBNAME := libsnappyjava.so
Linux-s390_SNAPPY_FLAGS :=
Linux-s390x_CXX := $(CROSS_PREFIX)g++
Linux-s390x_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
Linux-s390x_CXXFLAGS := -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64 -std=c++11
endif
Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390x_LIBNAME := libsnappyjava.so
Linux-s390x_SNAPPY_FLAGS :=
SunOS-x86_CXX := g++
SunOS-x86_CXX := g++
SunOS-x86_STRIP := strip
SunOS-x86_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
SunOS-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
SunOS-x86_LIBNAME := libsnappyjava.so
SunOS-x86_SNAPPY_FLAGS :=
SunOS-x86_SNAPPY_FLAGS :=
SunOS-sparc_CXX := g++
SunOS-sparc_CXX := g++
SunOS-sparc_STRIP := strip
SunOS-sparc_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
SunOS-sparc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
SunOS-sparc_LIBNAME := libsnappyjava.so
SunOS-sparc_SNAPPY_FLAGS :=
SunOS-sparc_SNAPPY_FLAGS :=
SunOS-x86_64_CXX := g++
SunOS-x86_64_STRIP := strip
@ -209,133 +144,66 @@ SunOS-x86_64_SNAPPY_FLAGS :=
Linux-arm_CXX := $(CROSS_PREFIX)g++
Linux-arm_STRIP := $(CROSS_PREFIX)strip
Linux-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp -std=c++11
Linux-arm_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp
Linux-arm_LINKFLAGS := -shared -static-libgcc
Linux-arm_LIBNAME := libsnappyjava.so
Linux-arm_SNAPPY_FLAGS:=
Linux-arm_SNAPPY_FLAGS:=
Linux-armv6_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++
Linux-armv6_STRIP := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip
Linux-armv6_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard -std=c++11
Linux-armv6_LINKFLAGS := -shared -static-libgcc -std=c++11
Linux-armv6_LIBNAME := libsnappyjava.so
Linux-armv6_SNAPPY_FLAGS:=
Linux-armv7_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++
Linux-armv7_STRIP := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip
Linux-armv7_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp -std=c++11
Linux-armv7_LINKFLAGS := -shared -static-libgcc
Linux-armv7_LIBNAME := libsnappyjava.so
Linux-armv7_SNAPPY_FLAGS:=
Linux-android-arm_CXX := $(CROSS_PREFIX)g++
Linux-android-arm_STRIP := $(CROSS_ROOT)/bin/llvm-strip
Linux-android-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-android-arm_LINKFLAGS := -shared -static-libgcc
Linux-android-arm_LIBNAME := libsnappyjava.so
Linux-android-arm_SNAPPY_FLAGS:=
Linux-android-aarch64_CXX := ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++
Linux-android-aarch64_STRIP := ${CROSS_ROOT}/bin/llvm-strip
Linux-android-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-android-aarch64_LINKFLAGS := -shared -static-libgcc
Linux-android-aarch64_LIBNAME := libsnappyjava.so
Linux-android-aarch64_SNAPPY_FLAGS:=
Linux-aarch64_CXX := $(CROSS_PREFIX)g++
Linux-aarch64_STRIP := $(CROSS_PREFIX)strip
Linux-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-aarch64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-aarch64_LIBNAME := libsnappyjava.so
Linux-aarch64_SNAPPY_FLAGS:=
Linux-armhf_CXX := $(CROSS_PREFIX)g++
Linux-armhf_STRIP := $(CROSS_PREFIX)strip
Linux-armhf_CXXFLAGS := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard
Linux-armhf_LINKFLAGS := -shared -static-libgcc
Linux-armhf_LIBNAME := libsnappyjava.so
Linux-armhf_SNAPPY_FLAGS:=
Mac-x86_CXX := g++ -arch i386
Mac-x86_STRIP := strip -x
Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden -std=c++11
Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden
Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc
Mac-x86_LIBNAME := libsnappyjava.dylib
Mac-x86_LIBNAME := libsnappyjava.jnilib
Mac-x86_SNAPPY_FLAGS :=
Mac-x86_64_CXX := c++ -arch $(OS_ARCH)
Mac-x86_64_CXX := g++ -arch $(OS_ARCH)
Mac-x86_64_STRIP := strip -x
Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11 -ftemplate-depth=1024
Mac-x86_64_LINKFLAGS := -dynamiclib
Mac-x86_64_LIBNAME := libsnappyjava.dylib
Mac-x86_64_SNAPPY_FLAGS :=
Mac-aarch64_CXX := c++ -arch arm64
Mac-aarch64_STRIP := strip -x
Mac-aarch64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11
Mac-aarch64_LINKFLAGS := -dynamiclib
Mac-aarch64_LIBNAME := libsnappyjava.dylib
Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden
Mac-x86_64_LINKFLAGS := -dynamiclib
Mac-x86_64_LIBNAME := libsnappyjava.jnilib
Mac-x86_64_SNAPPY_FLAGS :=
FreeBSD-x86_64_CXX := $(CROSS_PREFIX)g++
FreeBSD-x86_64_STRIP := $(CROSS_PREFIX)strip
FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11
FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
FreeBSD-x86_64_LINKFLAGS := -shared -static-libgcc
FreeBSD-x86_64_LIBNAME := libsnappyjava.so
FreeBSD-x86_64_SNAPPY_FLAGS :=
FreeBSD-x86_64_SNAPPY_FLAGS :=
Windows-x86_CXX := $(CROSS_PREFIX)g++
Windows-x86_STRIP := $(CROSS_PREFIX)strip
Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11
Windows-x86_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_LIBNAME := snappyjava.dll
Windows-x86_SNAPPY_FLAGS :=
Windows-x86_SNAPPY_FLAGS :=
Windows-x86_64_CXX := $(CROSS_PREFIX)g++
Windows-x86_64_STRIP := $(CROSS_PREFIX)strip
Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11
Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2
Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_64_LIBNAME := snappyjava.dll
Windows-x86_64_SNAPPY_FLAGS :=
Linux-riscv_CXX := $(CROSS_PREFIX)g++
Linux-riscv_STRIP := $(CROSS_PREFIX)strip
Linux-riscv_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-riscv_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-riscv_LIBNAME := libsnappyjava.so
Linux-riscv_SNAPPY_FLAGS:=
Linux-riscv64_CXX := $(CROSS_PREFIX)g++
Linux-riscv64_STRIP := $(CROSS_PREFIX)strip
Linux-riscv64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-riscv64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-riscv64_LIBNAME := libsnappyjava.so
Linux-riscv64_SNAPPY_FLAGS:=
Windows-aarch64_CXX := $(CROSS_PREFIX)g++
Windows-aarch64_STRIP := $(CROSS_PREFIX)strip
Windows-aarch64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -march=armv8-a -fvisibility=hidden -D_WIN32 -DNOMINMAX -std=c++11
Windows-aarch64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-aarch64_LIBNAME := snappyjava.dll
Linux-loongarch_CXX := $(CROSS_PREFIX)g++
Linux-loongarch_STRIP := $(CROSS_PREFIX)strip
Linux-loongarch_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-loongarch_LINKFLAGS := -shared
Linux-loongarch_LIBNAME := libsnappyjava.so
Linux-loongarch_SNAPPY_FLAGS:=
Linux-loongarch64_CXX := $(CROSS_PREFIX)g++
Linux-loongarch64_STRIP := $(CROSS_PREFIX)strip
Linux-loongarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
Linux-loongarch64_LINKFLAGS := -shared
Linux-loongarch64_LIBNAME := libsnappyjava.so
Linux-loongarch64_SNAPPY_FLAGS:=
Windows-x86_64_SNAPPY_FLAGS :=
CXX := $($(os_arch)_CXX)
STRIP := $($(os_arch)_STRIP)
CXXFLAGS := $($(os_arch)_CXXFLAGS)
LINKFLAGS := $($(os_arch)_LINKFLAGS)
CXXFLAGS := $($(os_arch)_CXXFLAGS)
LINKFLAGS := $($(os_arch)_LINKFLAGS)
LIBNAME := $($(os_arch)_LIBNAME)
SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS)
CXXFLAGS := $(CXXFLAGS) -DHAVE_CONFIG_H -DNDEBUG
CXXFLAGS := $(CXXFLAGS) -Ilib/include
ifneq ($(jni_include),)
CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
endif

View File

@ -28,8 +28,8 @@ DLL_DIR=src/main/resources/org/xerial/snappy/native
DLL_WIN=$(DLL_DIR)/Windows/x86/snappyjava.dll
DLL_WIN64=$(DLL_DIR)/Windows/amd64/snappyjava.dll
DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.dylib
DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.dylib
DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.jnilib
DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.jnilib
DLL_LINUX=$(DLL_DIR)/Linux/i386/libsnappyjava.so
DLL_AMD64=$(DLL_DIR)/Linux/amd64/libsnappyjava.so

View File

@ -1,128 +1,11 @@
## Features under consideration
* `SnappyIndexer` for parallel compression/decompression
* CUI commands (snap/unsnap)
Since snappy-java 1.1.9.0, release notes will be available in the GitHub releases page https://github.com/xerial/snappy-java/releases
Since vesion 1.1.0.x, Java 6 (1.6) or higher is required.
Since version 1.1.0.x, Java 6 (1.6) or higher is required.
## snappy-java-1.1.8.3 (2021-01-20)
* Make pure-java Snappy thread-safe [#271](https://github.com/xerial/snappy-java/pull/271)
* Improved SnappyFramedInput/OutputStream performance by using java.util.zip.CRC32C [#269](https://github.com/xerial/snappy-java/pull/269)
## snappy-java-1.1.8.2 (2020-11-28)
* Support Apple Silicon (M1, Mac-aarch64)
* Fixed the pure-java Snappy fallback logic when no native library for your platform is found.
## snappy-java-1.1.8.1 (2020-11-09)
* Fixed an initialization issue when using a recent Mac OS X version [#265](https://github.com/xerial/snappy-java/pull/265)
## snappy-java-1.1.8 (2020-10-20)
* Upgrade to [Snappy 1.1.8](https://github.com/google/snappy/releases/tag/1.1.8) with small performance improvements.
## snappy-java-1.1.7.8 (2020-10-20)
* Big-endian support for pure-java Snappy implementation
* linux-aarch64 (arm64) binary embeds libstdc++ for portability
* internal: Fix make native-all target to support the latest version of dockcross
## snappy-java-1.1.7.7 (2020-08-25)
* Built snappy-java with jdk8 to resolve #251 (java.lang.NoSuchMethodError)
## snappy-java-1.1.7.6 (2020-06-26)
* Added an experimental support of pure-java Snappy https://github.com/xerial/snappy-java#using-pure-java-snappy-implementation
* Pure-java snappy doesn't support Snappy.isValidCompressedBuffer methods, but the other methods, Snappy.compress, uncompress, SnappyInput/OutputStream, SnappyFramedInput/OutputStream, etc., should work as expected.
* Changed the minimum JVM requirement to JDK 1.8
## snappy-java-1.1.7.5 (2020-05-06)
* Fixes java.lang.NoClassDefFoundError: org/xerial/snappy/pool/DefaultPoolFactory in 1.1.7.4
## snapy-java-1.1.7.4 (2020-05-05)
* __DO NOT USE THIS VERSION__ 1.1.7.4 misses a package for using SnappyFramed streams.
* Caching internal buffers for SnappyFramed streams [#234](https://github.com/xerial/snappy-java/pull/234)
* Fixed the native lib for ppc64le to work with glibc 2.17 (Previously it depended on 2.22)
## snappy-java-1.1.7.3 (2019-03-25)
* Minor release
* Output the snappy header even for the empty input to address Spark's [issue](https://issues.apache.org/jira/browse/SPARK-27267)
* Fixed SnappyFramed stream to support Java 9
## snappy-java-1.1.7.2 (2018-05-21)
* Fix for aarch64 endian issue
## snappy-java-1.1.7.1 (2017-12-07)
* Fix for Android. No need to upgrade to this version if you are not using Android
## snappy-java-1.1.7 (2017-11-30)
* Upgrade to [snappy-1.1.7](https://github.com/google/snappy/releases/tag/1.1.7)
* Improved performance for big-endian architecture
* The other performance improvement in [snappy-1.1.5](https://github.com/google/snappy/releases/tag/1.1.5)
* (internal) Changed to use docker + cmake for building native libraries
* Fix android architecture resolution
* Add hadoop compatible SnappyHadoopCompatibleOutputStream
## snappy-java-1.1.4 (2017-05-22)
* Upgrade to [snappy-1.1.4](https://github.com/google/snappy/releases/tag/1.1.4)
* Improved performance compression (5%), decompression (20%)
* Added BitShuffle API for fast and better integer and floating-point value compresssion
* Added native libraries for s390x, AIX/Linux ppc64/ppc64le
* Added prelimirary support Linux armv5, armv6, armv7, android-arm, aarch64
* Using docker for cross-compiling native libraries for various platforms
* Note: snappy-java for FreeBSD, SunOS are still using snappy 1.1.2. Your contributions of native libraries are welcome. Please send a pull request that contains the changes after running `make native test` in your platform.
## snappy-java-1.1.4-M3 (2017-02-16)
* Fix native libraries for ppc64 (bigendian) and ppc64le (little endian)
## snappy-java-1.1.4-M2 (2017-02-13)
* Added s390x and AIX ppc/ppc64 support.
## snappy-java-1.1.4-M1 (2017-02-10)
* A preview release for 1.1.4
## snappy-java-1.1.3-M2 (2017-01-25)
* Rename to BitShuffle.shuffle/unshuffle
* Add ByteBuffer based APIs for bitshffle
## snappy-java-1.1.3-M1 (2017-01-19)
* This is a preview release. Appreciate your feedback.
* Upgraded to snappy 1.1.3 (Minor compression performance improvement)
* Added support for armv5, armv6, armv7, android-arm, aarch64, ppc64
* Added BitShuffle (<https://github.com/kiyo-masui/bitshuffle>) class for better primitive array compression
* Using docker-based cross compilers for building native libraries
* AIX, FreeBSD, SunOS, IBM s390x are still using snappy 1.1.2. Your contributions of native libraries are welcome. Please send a pull request that contains the changes after running `make native` in your platform.
## snappy-java-1.1.2.6 (2016-06-02)
* Fix a bug in SnappyInputStream when reading compressed data that happened to have the same first byte with the stream magic header (#142)
* Never use 1.1.2.5 (This is a broken build)
## snappy-java-1.1.2.4 (2016-03-30)
* Improved `SnappyOutputStream.write(byte[], off, len)` performance.
## snappy-java-1.1.2.3 (2016-03-30)
* Improved `SnappyInputStream.read(byte[], off, len)` performance.
## snappy-java-1.1.2.2 (2016-03-29)
* Add AArch64 support
## snappy-java-1.1.2.1 (2016-01-22)
* Fixed #131
## snappy-java-1.1.2 (22 September 2015)
* This is a backward compatible release for 1.1.x.
* Add AIX (32-bit) support.
* There is no upgrade for the native libraries of the other platforms.
* A major change since 1.1.1 is a support for reading concatenated results of SnappyOutputStream(s)
## snappy-java-1.1.2-RC2 (18 May 2015)
* Fix #107: SnappyOutputStream.close() is not idempotent
## snappy-java-1.1.2-RC1 (13 May 2015)
* SnappyInputStream now supports reading concatenated compressed results of SnappyOutputStream
* There has been no compressed format change since 1.0.5.x. So You can read the compressed results interchangeably between these versions.
* Fixes a problem when java.io.tmpdir does not exist.
## snappy-java-1.1.1.7 (14 Apr 2015)
* Fixes #100
## snappy-java-1.1.1.6 (26 Oct 2014)
* Fixes #88, #89, #90 and #91
* Fixed the broken build of 1.1.1.4 and memory leak bug 1.1.1.5 (so never use these versions)
## snappy-java-1.1.1.4 (23 Oct 2014)
* Fixes #88 and #89
## snappy-java-1.0.5.4 (12 September 2014)
* Embedded libstdc++ for Linux/amd64 native library (hotfix to 1.0.5.x series)
@ -144,7 +27,7 @@ Since version 1.1.0.x, Java 6 (1.6) or higher is required.
* Building with sbt
# snappy-java-1.0.5.2 (25 June 2014)
* Maintenance release
* Maintanance release
* Add PowerPC support ppc64le, ppc64 (big-endian)
# snappy-java-1.1.1-M2 (12 June 2014)
@ -198,7 +81,7 @@ Since version 1.1.0.x, Java 6 (1.6) or higher is required.
## snappy-java-1.0.3.2 (August 23rd, 2011)
* Upgrade from the previous release is optional
* Add system properties to switch system installed native library or bundled
* Add system properites to switch sytem installed native library or bundled
library (issue 20, issue 26)
* source code tar ball (issue 25)
* primitive array read/write support (issue 24)
@ -209,7 +92,7 @@ Since version 1.1.0.x, Java 6 (1.6) or higher is required.
* Rebuild win32 binary
## snappy-java-1.0.3 (July 11st, 2011)
* Deprecated SnappyException. Instead snappy-java uses IOException to issue
* Deprected SnappyException. Instead snappy-java uses IOException to issue
errors. This change is necessary to support JNI code injection to a parent
class loader.

9
NOTICE
View File

@ -5,8 +5,8 @@ This product includes software developed by Apache
PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/
(Apache 2.0 license)
This library contains statically linked libstdc++. This inclusion is allowed by
"GCC Runtime Library Exception"
This library containd statically linked libstdc++. This inclusion is allowed by
"GCC RUntime Library Exception"
http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
== Contributors ==
@ -15,8 +15,3 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
* Alec Wysoker
* Performance and memory usage improvement
Third-Party Notices and Licenses:
- Hadoop: Apache Hadoop is used as a dependency
License: Apache License 2.0
Source/Reference: https://github.com/apache/hadoop/blob/trunk/NOTICE.txt

260
README.md
View File

@ -1,47 +1,77 @@
snappy-java
[![Build Status](https://travis-ci.org/xerial/snappy-java.svg?branch=master)](https://travis-ci.org/xerial/snappy-java)
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/)
[![Javadoc](https://javadoc.io/badge2/org.xerial.snappy/snappy-java/javadoc.svg)](https://javadoc.io/doc/org.xerial.snappy/snappy-java)
===
snappy-java is a Java port of the [snappy](https://github.com/google/snappy), a fast C++ compresser/decompresser developed by Google.
The snappy-java is a Java port of the snappy
<http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google.
## Features
* Fast compression/decompression around 200~400MB/sec.
* Less memory usage. SnappyOutputStream uses only 32KB+ in default.
* JNI-based implementation to achieve comparable performance to the native C++ version.
## Features
* Fast compression/decompression tailored to 64-bit CPU architecture.
* JNI-based implementation to achieve comparable performance to the native C++ version.
* Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.).
* Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
* To improve the compression ratios of these arrays, you can use a fast data-rearrangement implementation ([`BitShuffle`](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.8/snappy-java-1.1.8-javadoc.jar/!/org/xerial/snappy/BitShuffle.html)) before compression
* Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux, etc. snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
* [Framing-format support](https://github.com/google/snappy/blob/master/framing_format.txt) (Since 1.1.0 version)
* Supporting compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
* Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
* [Framing-format support](http://snappy.googlecode.com/svn/trunk/framing_format.txt) (Since 1.1.0 version)
* OSGi support
* [Apache License Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). Free for both commercial and non-commercial use.
## Performance
## Performance
* Snappy's main target is very high-speed compression/decompression with reasonable compression size. So the compression ratio of snappy-java is modest and about the same as `LZF` (ranging 20%-100% according to the dataset).
* Here are some [benchmark results](https://github.com/ning/jvm-compressor-benchmark/wiki), comparing
snappy-java and the other compressors
`LZO-java`/`LZF`/`QuickLZ`/`Gzip`/`Bzip2`. Thanks [Tatu Saloranta @cotowncoder](http://twitter.com/#!/cowtowncoder) for providing the benchmark suite.
* The benchmark result indicates snappy-java is the fastest compressor/decompressor in Java: https://ning.github.io/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html
* The decompression speed is twice as fast as the others: https://ning.github.io/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html
`LZO-java`/`LZF`/`QuickLZ`/`Gzip`/`Bzip2`. Thanks [Tatu Saloranta @cotowncoder](http://twitter.com/#!/cowtowncoder) for providing the benchmark suite.
* The benchmark result indicates snappy-java is the fastest compreesor/decompressor in Java
* <http://ning.github.com/jvm-compressor-benchmark/results/canterbury-roundtrip-2011-07-28/index.html>
* The decompression speed is twice as fast as the others:
* <http://ning.github.com/jvm-compressor-benchmark/results/canterbury-uncompress-2011-07-28/index.html>
## Download
## Download
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/)
[![Javadoc](https://javadoc.io/badge2/org.xerial.snappy/snappy-java/javadoc.svg)](https://javadoc.io/doc/org.xerial.snappy/snappy-java)
* [Release Notes](Milestone.md)
* [Release Notes](Milestone.md)
The current stable version is available from here:
* Release version: https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/
* Release version: http://central.maven.org/maven2/org/xerial/snappy/snappy-java/
* Snapshot version (the latest beta version): https://oss.sonatype.org/content/repositories/snapshots/org/xerial/snappy/snappy-java/
For Maven user, see [pom.xml example](#using-with-maven).
## Usage
First, import `org.xerial.snapy.Snappy` in your Java code:
```java
import org.xerial.snappy.Snappy;
```
Then use `Snappy.compress(byte[])` and `Snappy.uncompress(byte[])`:
```java
String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of "
+ "Snappy, a fast compresser/decompresser.";
byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
byte[] uncompressed = Snappy.uncompress(compressed);
String result = new String(uncompressed, "UTF-8");
System.out.println(result);
```
In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used.
### Stream-based API
Stream-based compressor/decompressor `SnappyFramedOutputStream`/`SnappyFramedInputStream` are also available for reading/writing large data sets.
* See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.0/snappy-java-1.1.0-javadoc.jar/!/index.html)
### Setting classpath
If you have snappy-java-(VERSION).jar in the current directory, use `-classpath` option as follows:
$ javac -classpath ".;snappy-java-(VERSION).jar" Sample.java # in Windows
or
$ javac -classpath ".:snappy-java-(VERSION).jar" Sample.java # in Mac or Linux
### Using with Maven
Snappy-java is available from Maven's central repository. Add the following dependency to your pom.xml:
* Snappy-java is available from Maven's central repository: <http://repo1.maven.org/maven2/org/xerial/snappy/snappy-java>
Add the following dependency to your pom.xml:
<dependency>
<groupId>org.xerial.snappy</groupId>
@ -58,84 +88,65 @@ libraryDependencies += "org.xerial.snappy" % "snappy-java" % "(version)"
```
## Usage
First, import `org.xerial.snapy.Snappy` in your Java code:
```java
import org.xerial.snappy.Snappy;
```
Then use `Snappy.compress(byte[])` and `Snappy.uncompress(byte[])`:
```java
String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of "
+ "Snappy, a fast compresser/decompresser.";
byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
byte[] uncompressed = Snappy.uncompress(compressed);
String result = new String(uncompressed, "UTF-8");
System.out.println(result);
```
In addition, high-level methods (`Snappy.compress(String)`, `Snappy.compress(float[] ..)` etc. ) and low-level ones (e.g. `Snappy.rawCompress(.. )`, `Snappy.rawUncompress(..)`, etc.), which minimize memory copies, can be used.
### Stream-based API
Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` are also available for reading/writing large data sets. `SnappyFramedOutputStream`/`SnappyFramedInputStream` can be used for the [framing format](https://github.com/google/snappy/blob/master/framing_format.txt).
* See also [Javadoc API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.3-M1/snappy-java-1.1.3-M1-javadoc.jar/!/index.html)
#### Compatibility Notes
The original Snappy format definition did not define a file format. It later added
a "framing" format to define a file format, but by this point major software was
already using an industry standard instead -- represented in this library by the
`SnappyOutputStream` and `SnappyInputStream` methods.
For interoperability with other libraries, check that compatible formats are used.
Note that not all libraries support all variants.
* `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`.
* `SnappyHadoopCompatibleOutputStream` does not emit a file header but write out the current block size as a preemble to each block
#### Data format compatibility matrix:
| Write\Read | `Snappy.uncompress` | `SnappyInputStream` | `SnappyFramedInputStream` | `org.apache.hadoop.io.compress.SnappyCodec` |
| --------------- |:-------------------:|:------------------:|:-----------------------:|:-------------------------------------------:|
| `Snappy.compress` | ok | ok | x | x |
| `SnappyOutputStream` | x | ok | x | x |
| `SnappyFramedOutputStream` | x | x | ok | x |
| `SnappyHadoopCompatibleOutputStream` | x | x | x | ok |
### BitShuffle API (Since 1.1.3-M2)
BitShuffle is an algorithm that reorders data bits (shuffle) for efficient compression (e.g., a sequence of integers, float values, etc.). To use BitShuffle routines, import `org.xerial.snapy.BitShuffle`:
```java
import org.xerial.snappy.BitShuffle;
int[] data = new int[] {1, 3, 34, 43, 34};
byte[] shuffledByteArray = BitShuffle.shuffle(data);
byte[] compressed = Snappy.compress(shuffledByteArray);
byte[] uncompressed = Snappy.uncompress(compressed);
int[] result = BitShuffle.unshuffleIntArray(uncompress);
System.out.println(result);
```
Shuffling and unshuffling of primitive arrays (e.g., `short[]`, `long[]`, `float[]`, `double[]`, etc.) are supported. See [Javadoc](http://static.javadoc.io/org.xerial.snappy/snappy-java/1.1.3-M1/org/xerial/snappy/BitShuffle.html) for the details.
### Setting classpath
If you have snappy-java-(VERSION).jar in the current directory, use `-classpath` option as follows:
$ javac -classpath ".;snappy-java-(VERSION).jar" Sample.java # in Windows
or
$ javac -classpath ".:snappy-java-(VERSION).jar" Sample.java # in Mac or Linux
## Public discussion group
Post bug reports or feature request to the Issue Tracker: <https://github.com/xerial/snappy-java/issues>
Public discussion forum is here: [Xerial Public Discussion Group](http://groups.google.com/group/xerial?hl=en)
Public discussion forum is here: <http://groups.google.com/group/xerial?hl=en Xerial Public Discussion Group>
## Building from the source code
See the [installation instruction](https://github.com/xerial/snappy-java/blob/develop/INSTALL). Building from the source code is an option when your OS platform and CPU architecture is not supported. To build snappy-java, you need Git, JDK (1.6 or higher), g++ compiler (mingw in Windows) etc.
$ git clone https://github.com/xerial/snappy-java.git
$ cd snappy-java
$ make
When building on Solaris use
$ gmake
A file `target/snappy-java-$(version).jar` is the product additionally containing the native library built for your platform.
## Building linux x86_64 binary
snappy-java tries to static link libstdc++ to increase the availability for various Linux versions. However, standard distributions of 64-bit Linux OS rarely provide libstdc++ compiled with `-fPIC` option. I currently uses custom g++, compiled as follows:
```
$ cd work
$ wget (gcc-4.8.3 source)
$ tar xvfz (gcc-4.8.3.tar.gz)
$ cd gcc-4.8.3
$ ./contrib/download_prerequisites
$ cd ..
$ mkdir objdir
$ cd objdir
$ ../gcc-4.8.3/configure --prefix=$HOME/local/gcc-4.8.3 CXXFLAGS=-fPIC CFLAGS=-fPIC --enable-languages=c,c++
$ make
$ make install
```
This g++ build enables static linking of libstdc++. For more infomation on building GCC, see GCC's home page.
## Cross-compiling for other platforms
The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM:
$ make linux32 win32 win64 linux-arm linux-armhf
If you append `snappy` to the line above, it will also build the native library for the current platform and then build the snappy-java JAR (containing all native libraries built so far).
Of course, you must first have the necessary cross-compilers and development libraries installed for each target CPU and OS. For example, on Ubuntu 12.04 for x86-64, install the following packages for each target:
* linux32: `sudo apt-get install g++-multilib libc6-dev-i386 lib32stdc++6`
* win32: `sudo apt-get install g++-mingw-w64-i686`
* win64: `sudo apt-get install g++-mingw-w64-x86-64`
* arm: `sudo apt-get install g++-arm-linux-gnueabi`
* armhf: `sudo apt-get install g++-arm-linux-gnueabihf`
Unfortunately, cross-compiling for Mac OS X is not currently possible; you must compile within OS X.
If you are using Mac and openjdk7 (or higher), use the following option:
$ make native LIBNAME=libsnappyjava.dylib
## For developers
@ -143,55 +154,16 @@ snappy-java uses sbt (simple build tool for Scala) as a build tool. Here is a si
$ ./sbt # enter sbt console
> ~test # run tests upon source code change
> ~testOnly # run tests that matches a given name pattern
> ~test-only * # run tests that matches a given name pattern
> publishM2 # publish jar to $HOME/.m2/repository
> package # create jar file
> findbugs # Produce findbugs report in target/findbugs
> jacoco:cover # Report the code coverage of tests to target/jacoco folder
If you need to see detailed debug messages, launch sbt with `-Dloglevel=debug` option:
```
$ ./sbt -Dloglevel=debug
```
For the details of sbt usage, see my blog post: [Building Java Projects with sbt](http://xerial.org/blog/2014/03/24/sbt/)
### Building from the source code
See the [build instruction](https://github.com/xerial/snappy-java/blob/master/BUILD.md). Building from the source code is an option when your OS platform and CPU architecture is not supported. To build snappy-java, you need Git, JDK (1.6 or higher), g++ compiler (mingw in Windows) etc.
$ git clone https://github.com/xerial/snappy-java.git
$ cd snappy-java
$ make
When building on Solaris, use `gmake`:
$ gmake
A file `target/snappy-java-$(version).jar` is the product additionally containing the native library built for your platform.
### Creating a new release
GitHub action [https://github.com/xerial/snappy-java/blob/master/.github/workflows/release.yml] will publish a new relase to Maven Central (Sonatype) when a new tag vX.Y.Z is pushed.
## Miscellaneous Notes
### Using snappy-java with Tomcat 6 (or higher) Web Server
Simply put the snappy-java's jar to WEB-INF/lib folder of your web application. Usual JNI-library specific problem no longer exists since snappy-java version 1.0.3 or higher can be loaded by multiple class loaders.
### Configure snappy-java using property file
Prepare org-xerial-snappy.properties file (under the root path of your library) in Java's property file format.
Here is a list of the available properties:
* org.xerial.snappy.lib.path (directory containing a snappyjava's native library)
* org.xerial.snappy.lib.name (library file name)
* org.xerial.snappy.tempdir (temporary directory to extract a native library bundled in snappy-java)
* org.xerial.snappy.use.systemlib (if this value is true, use system installed libsnappyjava.so looking the path specified by java.library.path)
Simply put the snappy-java's jar to WEB-INF/lib folder of your web application. Usual JNI-library specific problem no longer exists since snappy-java version 1.0.3 or higher can be loaded by multiple class loaders.
----
Snappy-java is developed by [Taro L. Saito](http://www.xerial.org/leo). Twitter [@taroleo](http://twitter.com/#!/taroleo)

View File

@ -1,12 +0,0 @@
# Security Policy
## Supported Versions
| Version | Supported |
| ------- | ------------------ |
| 1.1.9.x | :white_check_mark: |
| < 1.1.8.x | :x: |
## Reporting a Vulnerability
https://github.com/xerial/snappy-java/security/advisories

185
build.sbt
View File

@ -1,124 +1,107 @@
Global / onChangedBuildSource := ReloadOnSourceChanges
import SonatypeKeys._
sonatypeSettings
name := "snappy-java"
organization := "org.xerial.snappy"
name := "snappy-java"
organization := "org.xerial.snappy"
organizationName := "xerial.org"
description := "snappy-java: A fast compression/decompression library"
sonatypeProfileName := "org.xerial"
ThisBuild / publishTo := sonatypePublishToBundle.value
licenses := Seq("Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.html"))
homepage := Some(url("https://github.com/xerial/snappy-java"))
scmInfo := Some(
ScmInfo(
browseUrl = url("https://github.com/xerial/snappy-java"),
connection = "scm:git@github.com:xerial/snappy-java.git"
)
)
developers := List(
Developer(id = "leo", name = "Taro L. Saito", email = "leo@xerial.org", url = url("http://xerial.org/leo"))
)
description := "snappy-java: A fast compression/decompression library"
// Use dynamic snapshot version strings for non tagged versions
ThisBuild / dynverSonatypeSnapshots := true
// Use coursier friendly version separator
ThisBuild / dynverSeparator := "-"
profileName := "org.xerial"
ThisBuild / scalaVersion := "2.12.11"
// For building jars for JDK8
ThisBuild / javacOptions ++= {
if (scala.util.Properties.isJavaAtLeast("9")) {
// --release 8 option is not available in JDK8
Seq("--release", "8")
} else {
Seq.empty
}
}
Compile / compile / javacOptions ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation")
doc / javacOptions := {
val opts = Seq("-source", "1.8")
if (scala.util.Properties.isJavaAtLeast("1.8"))
opts ++ Seq("-Xdoclint:none")
else
opts
pomExtra := {
<url>https://github.comm/xerial/snappy-java</url>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<developers>
<developer>
<id>leo</id>
<name>Taro L. Saito</name>
<email>leo@xerial.org</email>
<organization>Xerial Project</organization>
<roles>
<role>Architect</role>
<role>Project Manager</role>
<role>Chief Developer</role>
</roles>
<timezone>+9</timezone>
</developer>
</developers>
<issueManagement>
<system>GitHub</system>
<url>http://github.com/xerial/snappy-java/issues/list</url>
</issueManagement>
<inceptionYear>2011</inceptionYear>
<scm>
<connection>scm:git@github.com:xerial/snappy-java.git</connection>
<developerConnection>scm:git:git@github.com:xerial/snappy-java.git</developerConnection>
<url>git@github.com:xerial/snappy-java.git</url>
</scm>
}
// Configuration for SnappyHadoopCompatibleOutputStream testing
Test / fork := true
scalaVersion := "2.11.1"
val libTemp = {
val path = s"${System.getProperty("java.io.tmpdir")}/snappy_test_${System.currentTimeMillis()}"
// certain older Linux systems (debian/trusty in Travis CI) requires the libsnappy.so, loaded by
// libhadoop.so, be copied to the temp path before the child JVM is forked.
// because of that, cannot define as an additional task in Test scope
IO.copyFile(file("src/test/resources/lib/Linux/libsnappy.so"), file(s"$path/libsnappy.so"))
IO.copyFile(file("src/test/resources/lib/Linux/libsnappy.so"), file(s"$path/libsnappy.so.1"))
path
}
val macOSXLibPath = s"$libTemp:${System.getenv("DYLD_LIBRARY_PATH")}"
val linuxLibPath = s"$libTemp:${System.getenv("LD_LIBRARY_PATH")}"
// have to add to system dynamic library path since hadoop native library indirectly load libsnappy.1
// can't use javaOptions in Test because it causes the expression to eval twice yielding different temp path values
Test / envVars := Map("XERIAL_SNAPPY_LIB" -> libTemp, "DYLD_LIBRARY_PATH" -> macOSXLibPath, "LD_LIBRARY_PATH" -> linuxLibPath)
javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "1.6", "-target", "1.6")
testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v")
Test / parallelExecution := false
concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1))
autoScalaLibrary := false
crossPaths := false
crossPaths := false
logBuffered in Test := false
incOptions := incOptions.value.withNameHashing(true)
libraryDependencies ++= Seq(
"junit" % "junit" % "4.13.2" % "test",
"org.codehaus.plexus" % "plexus-classworlds" % "2.8.0" % "test",
"org.xerial.java" % "xerial-core" % "2.1" % "test",
"org.wvlet.airframe" %% "airframe-log" % "2025.1.1" % "test",
"org.osgi" % "org.osgi.core" % "6.0.0" % "provided",
"com.github.sbt" % "junit-interface" % "0.13.3" % "test",
"org.apache.hadoop" % "hadoop-common" % "3.4.1" % "test" exclude ("org.xerial.snappy", "snappy-java")
"junit" % "junit" % "4.8.2" % "test",
"org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test",
"org.xerial.java" % "xerial-core" % "2.1" % "test",
"org.xerial" % "xerial-core" % "3.2.3" % "test",
"org.scalatest" % "scalatest_2.11" % "2.2.0" % "test",
"org.osgi" % "org.osgi.core" % "4.3.0" % "provided",
"com.novocode" % "junit-interface" % "0.10" % "test"
)
enablePlugins(SbtOsgi)
osgiSettings
OsgiKeys.exportPackage := Seq("org.xerial.snappy", "org.xerial.snappy.buffer", "org.xerial.snappy.pool")
OsgiKeys.exportPackage := Seq("org.xerial.snappy", "org.xerial.snappy.buffer")
OsgiKeys.bundleSymbolicName := "org.xerial.snappy.snappy-java"
OsgiKeys.bundleActivator := Option("org.xerial.snappy.SnappyBundleActivator")
OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""")
OsgiKeys.requireCapability := """osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=1.7))""""
OsgiKeys.bundleActivator := Option("org.xerial.snappy.SnappyBundleActivator")
OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""")
OsgiKeys.additionalHeaders := Map(
"Bundle-NativeCode" -> Seq(
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64",
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64",
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64",
"org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
"org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
"org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib;osname=macosx;processor=x86-64",
"org/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib;osname=macosx;processor=aarch64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64",
"org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
"org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64",
"org/xerial/snappy/native/Linux/riscv64/libsnappyjava.so;osname=linux;processor=riscv64",
"org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
"org/xerial/snappy/native/Linux/armv7/libsnappyjava.so;osname=linux;processor=arm_le",
"org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64le",
"org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x",
"org/xerial/snappy/native/Linux/loongarch64/libsnappyjava.so;osname=linux;processor=loongarch64",
"org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc",
"org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
"org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
"org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
"org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc"
).mkString(","),
"Bundle-DocURL" -> "http://www.xerial.org/",
"Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt",
"Bundle-ActivationPolicy" -> "lazy",
"Bundle-Name" -> "snappy-java: A fast compression/decompression library"
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64",
"org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
"org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
"org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
"org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
"org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
"org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64",
"org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so;osname=linux;processor=ppc64le",
"org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
"org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
"org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
"org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc"
).mkString(","),
"Bundle-DocURL" -> "http://www.xerial.org/",
"Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt",
"Bundle-ActivationPolicy" -> "lazy",
"Bundle-Name" -> "snappy-java: A fast compression/decompression library"
)

View File

@ -1,41 +0,0 @@
FROM centos:5
MAINTAINER Taro L. Saito <leo@xerial.org>
RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
RUN sed -i 's/mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo
RUN sed -i 's|#baseurl=http://mirror.centos.org/centos/$releasever|baseurl=http://vault.centos.org/5.11|' /etc/yum.repos.d/*.repo
RUN yum -y install make gcc gcc-c++ glibc-devel perl wget bzip2 curl \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir /tmp/work \
&& cd /tmp/work \
&& wget http://www.netgull.com/gcc/releases/gcc-4.8.3/gcc-4.8.3.tar.gz \
&& tar xvfz gcc-4.8.3.tar.gz \
&& cd gcc-4.8.3 \
&& ./contrib/download_prerequisites \
&& cd .. \
&& mkdir objdir
RUN cd /tmp/work/objdir \
&& ../gcc-4.8.3/configure --prefix=/usr/local/gcc-4.8.3 CXXFLAGS=-fPIC CFLAGS=-fPIC --enable-languages=c,c++ \
&& make
RUN cd /tmp/work/objdir \
&& make install \
&& rm -rf /tmp/work
#RUN mkdir /tmp/cmake \
# && cd /tmp/cmake \
# && wget --no-check-certificate https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz \
# && tar xvfz cmake-3.10.0.tar.gz \
# && cd cmake-3.10.0 \
# && CXX=/usr/local/gcc-4.8.3/bin/g++ ./bootstrap \
# && make \
# && make install \
# && rm -rf /tmp/cmake
ENV PATH /usr/local/gcc-4.8.3/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/gcc-4.8.3/lib64/:$LD_LIBRARY_PATH
WORKDIR /work

View File

@ -1,55 +0,0 @@
FROM dockcross/base:latest
MAINTAINER Taro L. Saito "leo@xerial.org"
# Add the cross compiler sources
RUN echo "deb http://emdebian.org/tools/debian/ jessie main" >> /etc/apt/sources.list && \
dpkg --add-architecture powerpc && \
curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add -
RUN apt-get update && apt-get install -y \
crossbuild-essential-powerpc \
gfortran-powerpc-linux-gnu \
libbz2-dev:powerpc \
libexpat1-dev:powerpc \
ncurses-dev:powerpc \
libssl-dev:powerpc
WORKDIR /usr/src
RUN apt-get update && \
apt-get install -y libglib2.0-dev zlib1g-dev libpixman-1-dev && \
curl -L http://wiki.qemu-project.org/download/qemu-2.6.0.tar.bz2 | tar xj && \
cd qemu-2.6.0 && \
./configure --target-list=ppc64-linux-user --prefix=/usr && \
make -j$(nproc) && \
make install && \
cd .. && rm -rf qemu-2.6.0
ENV CROSS_TRIPLE powerpc-linux-gnu
ENV CROSS_ROOT /usr/${CROSS_TRIPLE}
ENV AS=/usr/bin/${CROSS_TRIPLE}-as \
AR=/usr/bin/${CROSS_TRIPLE}-ar \
CC=/usr/bin/${CROSS_TRIPLE}-gcc \
CPP=/usr/bin/${CROSS_TRIPLE}-cpp \
CXX=/usr/bin/${CROSS_TRIPLE}-g++ \
LD=/usr/bin/${CROSS_TRIPLE}-ld
ENV DEFAULT_DOCKCROSS_IMAGE dockcross/linux-ppc64
WORKDIR /work
# Note: Toolchain file support is currently in debian Experimental according to:
# https://wiki.debian.org/CrossToolchains#In_jessie_.28Debian_8.29
# We can switch to that when it becomes stable.
COPY Toolchain-ppc64.cmake /usr/lib/${CROSS_TRIPLE}/Toolchain.cmake
ENV CMAKE_TOOLCHAIN_FILE /usr/lib/${CROSS_TRIPLE}/Toolchain.cmake
# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
ARG VCS_REF
ARG VCS_URL
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name=$IMAGE \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url=$VCS_URL \
org.label-schema.schema-version="1.0"

View File

@ -1,9 +0,0 @@
centos5-image:
docker build -f Dockerfile.centos5-linux-x86_64-pic -t xerial/centos5-linux-x86_64:latest
# dockerhub login
# docker login --username=xerial
# docker push xerial/centos5-linux-x86_64:latest
multiarch-crossbuild-image:
docker build https://github.com/iwasakims/crossbuild.git#fix-osxcross-cmake -t xerial/crossbuild:latest

View File

@ -1,15 +0,0 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR ppc64)
set(cross_triple "ppc64-linux-gnu")
set(CMAKE_C_COMPILER /usr/bin/${cross_triple}-cc)
set(CMAKE_CXX_COMPILER /usr/bin/${cross_triple}-c++)
set(CMAKE_Fortran_COMPILER /usr/bin/${cross_triple}-gfortran)
# Discard path returned by pkg-config and associated with HINTS in module
# like FindOpenSSL.
set(CMAKE_IGNORE_PATH /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/lib/)
set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-ppc64)

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64-lts
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,279 +0,0 @@
#!/usr/bin/env bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm:20230116-670f7f7
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 "ERROR: $*\n"
}
die() {
err "$*"
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative")
if [ -z "$OCI_EXE" ]; then
if which podman >/dev/null 2>/dev/null; then
OCI_EXE=podman
elif which docker >/dev/null 2>/dev/null; then
OCI_EXE=docker
else
die "Cannot find a container executor. Search for docker and podman."
fi
fi
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
$OCI_EXE pull $FINAL_IMAGE
}
help:update-image() {
echo "Pull the latest $FINAL_IMAGE ."
}
command:update-script() {
if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then
echo "$0 is up to date"
else
echo -n "Updating $0 ... "
$OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-script() {
echo "Update $0 from $FINAL_IMAGE ."
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo "Pull the latest $FINAL_IMAGE, and then update $0 from that."
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
shift
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# Bash on Ubuntu on Windows
UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
# CYGWIN
CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "")
if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then
USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
fi
# Change the PWD when working in Docker on Windows
if [ -n "$UBUNTU_ON_WINDOWS" ]; then
WSL_ROOT="/mnt/"
CFG_FILE=/etc/wsl.conf
if [ -f "$CFG_FILE" ]; then
CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
eval "$CFG_CONTENT"
if [ -n "$root" ]; then
WSL_ROOT=$root
fi
fi
HOST_PWD=`pwd -P`
HOST_PWD=${HOST_PWD/$WSL_ROOT//}
elif [ -n "$MSYS" ]; then
HOST_PWD=$PWD
HOST_PWD=${HOST_PWD/\//}
HOST_PWD=${HOST_PWD/\//:\/}
elif [ -n "$CYGWIN" ]; then
for f in pwd readlink cygpath ; do
test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ;
done ;
HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ;
else
HOST_PWD=$PWD
[ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD)
fi
# Mount Additional Volumes
if [ -z "$SSH_DIR" ]; then
SSH_DIR="$HOME/.ssh"
fi
HOST_VOLUMES=
if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then
if test -n "${CYGWIN}" ; then
HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ;
else
HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ;
fi ;
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
TTY_ARGS=
tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti
CONTAINER_NAME=dockcross_$RANDOM
$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \
--platform linux/amd64 \
-v "$HOST_PWD":/work \
$HOST_VOLUMES \
"${USER_IDS[@]}" \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
run_exit_code=$?
# Attempt to delete container
rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1)
rm_exit_code=$?
if [[ $rm_exit_code != 0 ]]; then
if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then
: # Ignore error because of https://circleci.com/docs/docker-btrfs-error/
else
echo "$rm_output"
exit $rm_exit_code
fi
fi
exit $run_exit_code
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/android-arm:20230116-670f7f7 image, run:
#
# docker run --rm dockcross/android-arm:20230116-670f7f7 > dockcross-android-arm-20230116-670f7f7
# chmod +x dockcross-android-arm-20230116-670f7f7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,278 +0,0 @@
#!/usr/bin/env bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/android-arm64:20230421-a0eaff4
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 "ERROR: $*\n"
}
die() {
err "$*"
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative")
if [ -z "$OCI_EXE" ]; then
if which podman >/dev/null 2>/dev/null; then
OCI_EXE=podman
elif which docker >/dev/null 2>/dev/null; then
OCI_EXE=docker
else
die "Cannot find a container executor. Search for docker and podman."
fi
fi
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
$OCI_EXE pull $FINAL_IMAGE
}
help:update-image() {
echo "Pull the latest $FINAL_IMAGE ."
}
command:update-script() {
if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then
echo "$0 is up to date"
else
echo -n "Updating $0 ... "
$OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-script() {
echo "Update $0 from $FINAL_IMAGE ."
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo "Pull the latest $FINAL_IMAGE, and then update $0 from that."
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
shift
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# Bash on Ubuntu on Windows
UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
# CYGWIN
CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "")
if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then
USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
fi
# Change the PWD when working in Docker on Windows
if [ -n "$UBUNTU_ON_WINDOWS" ]; then
WSL_ROOT="/mnt/"
CFG_FILE=/etc/wsl.conf
if [ -f "$CFG_FILE" ]; then
CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
eval "$CFG_CONTENT"
if [ -n "$root" ]; then
WSL_ROOT=$root
fi
fi
HOST_PWD=`pwd -P`
HOST_PWD=${HOST_PWD/$WSL_ROOT//}
elif [ -n "$MSYS" ]; then
HOST_PWD=$PWD
HOST_PWD=${HOST_PWD/\//}
HOST_PWD=${HOST_PWD/\//:\/}
elif [ -n "$CYGWIN" ]; then
for f in pwd readlink cygpath ; do
test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ;
done ;
HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ;
else
HOST_PWD=$PWD
[ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD)
fi
# Mount Additional Volumes
if [ -z "$SSH_DIR" ]; then
SSH_DIR="$HOME/.ssh"
fi
HOST_VOLUMES=
if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then
if test -n "${CYGWIN}" ; then
HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ;
else
HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ;
fi ;
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
TTY_ARGS=
tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti
CONTAINER_NAME=dockcross_$RANDOM
$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \
-v "$HOST_PWD":/work \
$HOST_VOLUMES \
"${USER_IDS[@]}" \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
run_exit_code=$?
# Attempt to delete container
rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1)
rm_exit_code=$?
if [[ $rm_exit_code != 0 ]]; then
if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then
: # Ignore error because of https://circleci.com/docs/docker-btrfs-error/
else
echo "$rm_output"
exit $rm_exit_code
fi
fi
exit $run_exit_code
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/android-arm64:20230421-a0eaff4 image, run:
#
# docker run --rm dockcross/android-arm64:20230421-a0eaff4 > dockcross-android-arm64-20230421-a0eaff4
# chmod +x dockcross-android-arm64-20230421-a0eaff4
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,242 +0,0 @@
#!/usr/bin/env bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64-lts
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run --rm $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run --rm $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
shift
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# Bash on Ubuntu on Windows
UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" ]; then
USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
fi
# Change the PWD when working in Docker on Windows
if [ -n "$UBUNTU_ON_WINDOWS" ]; then
WSL_ROOT="/mnt/"
CFG_FILE=/etc/wsl.conf
if [ -f "$CFG_FILE" ]; then
CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
eval "$CFG_CONTENT"
if [ -n "$root" ]; then
WSL_ROOT=$root
fi
fi
HOST_PWD=`pwd -P`
HOST_PWD=${HOST_PWD/$WSL_ROOT//}
elif [ -n "$MSYS" ]; then
HOST_PWD=$PWD
HOST_PWD=${HOST_PWD/\//}
HOST_PWD=${HOST_PWD/\//:\/}
else
HOST_PWD=$PWD
fi
# Mount Additional Volumes
if [ -z "$SSH_DIR" ]; then
SSH_DIR="$HOME/.ssh"
fi
HOST_VOLUMES=
if [ -e "$SSH_DIR" ]; then
HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
tty -s && TTY_ARGS=-ti || TTY_ARGS=
CONTAINER_NAME=dockcross_$RANDOM
docker run $TTY_ARGS --name $CONTAINER_NAME \
-v "$HOST_PWD":/work \
$HOST_VOLUMES \
"${USER_IDS[@]}" \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
run_exit_code=$?
exit $run_exit_code
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-arm64 image, run:
#
# docker run --rm dockcross/linux-arm64 > dockcross-linux-arm64
# chmod +x dockcross-linux-arm64
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv5
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv6-lts
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,278 +0,0 @@
#!/usr/bin/env bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-armv7-lts:20230421-a0eaff4
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 "ERROR: $*\n"
}
die() {
err "$*"
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative")
if [ -z "$OCI_EXE" ]; then
if which podman >/dev/null 2>/dev/null; then
OCI_EXE=podman
elif which docker >/dev/null 2>/dev/null; then
OCI_EXE=docker
else
die "Cannot find a container executor. Search for docker and podman."
fi
fi
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
$OCI_EXE pull $FINAL_IMAGE
}
help:update-image() {
echo "Pull the latest $FINAL_IMAGE ."
}
command:update-script() {
if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then
echo "$0 is up to date"
else
echo -n "Updating $0 ... "
$OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-script() {
echo "Update $0 from $FINAL_IMAGE ."
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo "Pull the latest $FINAL_IMAGE, and then update $0 from that."
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
shift
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# Bash on Ubuntu on Windows
UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
# MSYS, Git Bash, etc.
MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
# CYGWIN
CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "")
if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then
USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
fi
# Change the PWD when working in Docker on Windows
if [ -n "$UBUNTU_ON_WINDOWS" ]; then
WSL_ROOT="/mnt/"
CFG_FILE=/etc/wsl.conf
if [ -f "$CFG_FILE" ]; then
CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
eval "$CFG_CONTENT"
if [ -n "$root" ]; then
WSL_ROOT=$root
fi
fi
HOST_PWD=`pwd -P`
HOST_PWD=${HOST_PWD/$WSL_ROOT//}
elif [ -n "$MSYS" ]; then
HOST_PWD=$PWD
HOST_PWD=${HOST_PWD/\//}
HOST_PWD=${HOST_PWD/\//:\/}
elif [ -n "$CYGWIN" ]; then
for f in pwd readlink cygpath ; do
test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ;
done ;
HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ;
else
HOST_PWD=$PWD
[ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD)
fi
# Mount Additional Volumes
if [ -z "$SSH_DIR" ]; then
SSH_DIR="$HOME/.ssh"
fi
HOST_VOLUMES=
if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then
if test -n "${CYGWIN}" ; then
HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ;
else
HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ;
fi ;
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
TTY_ARGS=
tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti
CONTAINER_NAME=dockcross_$RANDOM
$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \
-v "$HOST_PWD":/work \
$HOST_VOLUMES \
"${USER_IDS[@]}" \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
run_exit_code=$?
# Attempt to delete container
rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1)
rm_exit_code=$?
if [[ $rm_exit_code != 0 ]]; then
if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then
: # Ignore error because of https://circleci.com/docs/docker-btrfs-error/
else
echo "$rm_output"
exit $rm_exit_code
fi
fi
exit $run_exit_code
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7-lts:20230421-a0eaff4 image, run:
#
# docker run --rm dockcross/linux-armv7-lts:20230421-a0eaff4 > dockcross-linux-armv7-lts-20230421-a0eaff4
# chmod +x dockcross-linux-armv7-lts-20230421-a0eaff4
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=xerial/linux-ppc64
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-ppc64le-lts
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-riscv64
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,199 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-s390x:20210210-84c47e5
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,199 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/windows-arm64
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/windows-x64
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

View File

@ -1,200 +0,0 @@
#!/bin/bash
DEFAULT_DOCKCROSS_IMAGE=dockcross/windows-x86
#------------------------------------------------------------------------------
# Helpers
#
err() {
echo -e >&2 ERROR: $@\\n
}
die() {
err $@
exit 1
}
has() {
# eg. has command update
local kind=$1
local name=$2
type -t $kind:$name | grep -q function
}
#------------------------------------------------------------------------------
# Command handlers
#
command:update-image() {
docker pull $FINAL_IMAGE
}
help:update-image() {
echo Pull the latest $FINAL_IMAGE .
}
command:update-script() {
if cmp -s <( docker run $FINAL_IMAGE ) $0; then
echo $0 is up to date
else
echo -n Updating $0 '... '
docker run $FINAL_IMAGE > $0 && echo ok
fi
}
help:update-image() {
echo Update $0 from $FINAL_IMAGE .
}
command:update() {
command:update-image
command:update-script
}
help:update() {
echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
}
command:help() {
if [[ $# != 0 ]]; then
if ! has command $1; then
err \"$1\" is not an dockcross command
command:help
elif ! has help $1; then
err No help found for \"$1\"
else
help:$1
fi
else
cat >&2 <<ENDHELP
Usage: dockcross [options] [--] command [args]
By default, run the given *command* in an dockcross Docker container.
The *options* can be one of:
--args|-a Extra args to the *docker run* command
--image|-i Docker cross-compiler image to use
--config|-c Bash script to source before running this script
Additionally, there are special update commands:
update-image
update-script
update
For update command help use: $0 help <command>
ENDHELP
exit 1
fi
}
#------------------------------------------------------------------------------
# Option processing
#
special_update_command=''
while [[ $# != 0 ]]; do
case $1 in
--)
break
;;
--args|-a)
ARG_ARGS="$2"
shift 2
;;
--config|-c)
ARG_CONFIG="$2"
shift 2
;;
--image|-i)
ARG_IMAGE="$2"
shift 2
;;
update|update-image|update-script)
special_update_command=$1
break
;;
-*)
err Unknown option \"$1\"
command:help
exit
;;
*)
break
;;
esac
done
# The precedence for options is:
# 1. command-line arguments
# 2. environment variables
# 3. defaults
# Source the config file if it exists
DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
# Set the docker image
FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
# Handle special update command
if [ "$special_update_command" != "" ]; then
case $special_update_command in
update)
command:update
exit $?
;;
update-image)
command:update-image
exit $?
;;
update-script)
command:update-script
exit $?
;;
esac
fi
# Set the docker run extra args (if any)
FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
# If we are not running via boot2docker
if [ -z $DOCKER_HOST ]; then
USER_IDS="-e BUILDER_UID=$( id -u ) -e BUILDER_GID=$( id -g ) -e BUILDER_USER=$( id -un ) -e BUILDER_GROUP=$( id -gn )"
fi
#------------------------------------------------------------------------------
# Now, finally, run the command in a container
#
docker run --rm \
-v $PWD:/work \
$USER_IDS \
$FINAL_ARGS \
$FINAL_IMAGE "$@"
################################################################################
#
# This image is not intended to be run manually.
#
# To create a dockcross helper script for the
# dockcross/linux-armv7 image, run:
#
# docker run --rm dockcross/linux-armv7 > dockcross-linux-armv7
# chmod +x dockcross-linux-armv7
#
# You may then wish to move the dockcross script to your PATH.
#
################################################################################

248
googlecode_upload.py Executable file
View File

@ -0,0 +1,248 @@
#!/usr/bin/env python
#
# Copyright 2006, 2007 Google Inc. All Rights Reserved.
# Author: danderson@google.com (David Anderson)
#
# Script for uploading files to a Google Code project.
#
# This is intended to be both a useful script for people who want to
# streamline project uploads and a reference implementation for
# uploading files to Google Code projects.
#
# To upload a file to Google Code, you need to provide a path to the
# file on your local machine, a small summary of what the file is, a
# project name, and a valid account that is a member or owner of that
# project. You can optionally provide a list of labels that apply to
# the file. The file will be uploaded under the same name that it has
# in your local filesystem (that is, the "basename" or last path
# component). Run the script with '--help' to get the exact syntax
# and available options.
#
# Note that the upload script requests that you enter your
# googlecode.com password. This is NOT your Gmail account password!
# This is the password you use on googlecode.com for committing to
# Subversion and uploading files. You can find your password by going
# to http://code.google.com/hosting/settings when logged in with your
# Gmail account. If you have already committed to your project's
# Subversion repository, the script will automatically retrieve your
# credentials from there (unless disabled, see the output of '--help'
# for details).
#
# If you are looking at this script as a reference for implementing
# your own Google Code file uploader, then you should take a look at
# the upload() function, which is the meat of the uploader. You
# basically need to build a multipart/form-data POST request with the
# right fields and send it to https://PROJECT.googlecode.com/files .
# Authenticate the request using HTTP Basic authentication, as is
# shown below.
#
# Licensed under the terms of the Apache Software License 2.0:
# http://www.apache.org/licenses/LICENSE-2.0
#
# Questions, comments, feature requests and patches are most welcome.
# Please direct all of these to the Google Code users group:
# http://groups.google.com/group/google-code-hosting
"""Google Code file uploader script.
"""
__author__ = 'danderson@google.com (David Anderson)'
import httplib
import os.path
import optparse
import getpass
import base64
import sys
def upload(file, project_name, user_name, password, summary, labels=None):
"""Upload a file to a Google Code project's file server.
Args:
file: The local path to the file.
project_name: The name of your project on Google Code.
user_name: Your Google account name.
password: The googlecode.com password for your account.
Note that this is NOT your global Google Account password!
summary: A small description for the file.
labels: an optional list of label strings with which to tag the file.
Returns: a tuple:
http_status: 201 if the upload succeeded, something else if an
error occured.
http_reason: The human-readable string associated with http_status
file_url: If the upload succeeded, the URL of the file on Google
Code, None otherwise.
"""
# The login is the user part of user@gmail.com. If the login provided
# is in the full user@domain form, strip it down.
if user_name.endswith('@gmail.com'):
user_name = user_name[:user_name.index('@gmail.com')]
form_fields = [('summary', summary)]
if labels is not None:
form_fields.extend([('label', l.strip()) for l in labels])
content_type, body = encode_upload_request(form_fields, file)
upload_host = '%s.googlecode.com' % project_name
upload_uri = '/files'
auth_token = base64.b64encode('%s:%s'% (user_name, password))
headers = {
'Authorization': 'Basic %s' % auth_token,
'User-Agent': 'Googlecode.com uploader v0.9.4',
'Content-Type': content_type,
}
server = httplib.HTTPSConnection(upload_host)
server.request('POST', upload_uri, body, headers)
resp = server.getresponse()
server.close()
if resp.status == 201:
location = resp.getheader('Location', None)
else:
location = None
return resp.status, resp.reason, location
def encode_upload_request(fields, file_path):
"""Encode the given fields and file into a multipart form body.
fields is a sequence of (name, value) pairs. file is the path of
the file to upload. The file will be uploaded to Google Code with
the same file name.
Returns: (content_type, body) ready for httplib.HTTP instance
"""
BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
CRLF = '\r\n'
body = []
# Add the metadata about the upload first
for key, value in fields:
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"' % key,
'',
value,
])
# Now add the file itself
file_name = os.path.basename(file_path)
f = open(file_path, 'rb')
file_content = f.read()
f.close()
body.extend(
['--' + BOUNDARY,
'Content-Disposition: form-data; name="filename"; filename="%s"'
% file_name,
# The upload server determines the mime-type, no need to set it.
'Content-Type: application/octet-stream',
'',
file_content,
])
# Finalize the form body
body.extend(['--' + BOUNDARY + '--', ''])
return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
def upload_find_auth(file_path, project_name, summary, labels=None,
user_name=None, password=None, tries=3):
"""Find credentials and upload a file to a Google Code project's file server.
file_path, project_name, summary, and labels are passed as-is to upload.
Args:
file_path: The local path to the file.
project_name: The name of your project on Google Code.
summary: A small description for the file.
labels: an optional list of label strings with which to tag the file.
config_dir: Path to Subversion configuration directory, 'none', or None.
user_name: Your Google account name.
tries: How many attempts to make.
"""
while tries > 0:
if user_name is None:
# Read username if not specified or loaded from svn config, or on
# subsequent tries.
sys.stdout.write('Please enter your googlecode.com username: ')
sys.stdout.flush()
user_name = sys.stdin.readline().rstrip()
if password is None:
# Read password if not loaded from svn config, or on subsequent tries.
print 'Please enter your googlecode.com password.'
print '** Note that this is NOT your Gmail account password! **'
print 'It is the password you use to access Subversion repositories,'
print 'and can be found here: http://code.google.com/hosting/settings'
password = getpass.getpass()
status, reason, url = upload(file_path, project_name, user_name, password,
summary, labels)
# Returns 403 Forbidden instead of 401 Unauthorized for bad
# credentials as of 2007-07-17.
if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]:
# Rest for another try.
user_name = password = None
tries = tries - 1
else:
# We're done.
break
return status, reason, url
def main():
parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY '
'-p PROJECT [options] FILE')
parser.add_option('-s', '--summary', dest='summary',
help='Short description of the file')
parser.add_option('-p', '--project', dest='project',
help='Google Code project name')
parser.add_option('-u', '--user', dest='user',
help='Your Google Code username')
parser.add_option('-w', '--password', dest='password',
help='Your Google Code password')
parser.add_option('-l', '--labels', dest='labels',
help='An optional list of comma-separated labels to attach '
'to the file')
options, args = parser.parse_args()
if not options.summary:
parser.error('File summary is missing.')
elif not options.project:
parser.error('Project name is missing.')
elif len(args) < 1:
parser.error('File to upload not provided.')
elif len(args) > 1:
parser.error('Only one file may be specified.')
file_path = args[0]
if options.labels:
labels = options.labels.split(',')
else:
labels = None
status, reason, url = upload_find_auth(file_path, options.project,
options.summary, labels,
options.user, options.password)
if url:
print 'The file was uploaded successfully.'
print 'URL: %s' % url
return 0
else:
print 'An error occurred. Your file was not uploaded.'
print 'Google Code upload server said: %s (%s)' % (reason, status)
return 1
if __name__ == '__main__':
sys.exit(main())

View File

@ -1,62 +0,0 @@
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
/* Define to 1 if the compiler supports __builtin_ctz and friends. */
#define HAVE_BUILTIN_CTZ 1
/* Define to 1 if the compiler supports __builtin_expect. */
#define HAVE_BUILTIN_EXPECT 1
/* Define to 1 if you have the <byteswap.h> header file. */
/* #undef HAVE_BYTESWAP_H */
/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
#define HAVE_FUNC_MMAP 1
/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
#define HAVE_FUNC_SYSCONF 1
/* Define to 1 to use the gflags package for command-line parsing. */
/* #undef HAVE_GFLAGS */
/* Define to 1 if you have Google Test. */
/* #undef HAVE_GTEST */
/* Define to 1 if you have the `lzo2' library (-llzo2). */
/* #undef HAVE_LIBLZO2 */
/* Define to 1 if you have the `z' library (-lz). */
#define HAVE_LIBZ 1
/* Define to 1 if you have the <stddef.h> header file. */
#define HAVE_STDDEF_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <sys/endian.h> header file. */
/* #undef HAVE_SYS_ENDIAN_H */
/* Define to 1 if you have the <sys/mman.h> header file. */
#define HAVE_SYS_MMAN_H 1
/* Define to 1 if you have the <sys/resource.h> header file. */
#define HAVE_SYS_RESOURCE_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Define to 1 if you have the <windows.h> header file. */
/* #undef HAVE_WINDOWS_H */
/* Define to 1 if your processor stores words with the most significant byte
first (like Motorola and SPARC, unlike Intel and VAX). */
/* #undef SNAPPY_IS_BIG_ENDIAN */
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_

View File

@ -1,26 +1,26 @@
/*
* Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
* ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
*
* Copyright (c) 1996, 2006, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
@ -1951,10 +1951,9 @@ JNI_OnUnload(JavaVM *vm, void *reserved);
#define JNI_VERSION_1_2 0x00010002
#define JNI_VERSION_1_4 0x00010004
#define JNI_VERSION_1_6 0x00010006
#define JNI_VERSION_1_8 0x00010008
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
#endif /* !_JAVASOFT_JNI_H_ */
#endif /* !_JAVASOFT_JNI_H_ */

View File

@ -1,94 +0,0 @@
// Copyright 2011 Google Inc. All Rights Reserved.
// Author: sesse@google.com (Steinar H. Gunderson)
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Various type stubs for the open-source version of Snappy.
//
// This file cannot include config.h, as it is included from snappy.h,
// which is a public header. Instead, snappy-stubs-public.h is generated by
// from snappy-stubs-public.h.in at configure time.
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
#if 1 // HAVE_STDINT_H
#include <stdint.h>
#endif // HAVE_STDDEF_H
#if 1 // HAVE_STDDEF_H
#include <stddef.h>
#endif // HAVE_STDDEF_H
#if 1 // HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif // HAVE_SYS_UIO_H
#define SNAPPY_MAJOR
#define SNAPPY_MINOR
#define SNAPPY_PATCHLEVEL
#define SNAPPY_VERSION \
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
#include <string>
namespace snappy {
#if 1 // HAVE_STDINT_H
typedef int8_t int8;
typedef uint8_t uint8;
typedef int16_t int16;
typedef uint16_t uint16;
typedef int32_t int32;
typedef uint32_t uint32;
typedef int64_t int64;
typedef uint64_t uint64;
#else
typedef signed char int8;
typedef unsigned char uint8;
typedef short int16;
typedef unsigned short uint16;
typedef int int32;
typedef unsigned int uint32;
typedef long long int64;
typedef unsigned long long uint64;
#endif // HAVE_STDINT_H
typedef std::string string;
#if !1 // !HAVE_SYS_UIO_H
// Windows does not have an iovec type, yet the concept is universally useful.
// It is simple to define it ourselves, so we put it inside our own namespace.
struct iovec {
void* iov_base;
size_t iov_len;
};
#endif // !HAVE_SYS_UIO_H
} // namespace snappy
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_

5
lib/include/config.h Executable file
View File

@ -0,0 +1,5 @@
#ifndef __CONFIG_H
#define __CONFIG_H
#endif // __CONFIG_H

Binary file not shown.

View File

@ -1,2 +1,2 @@
sbt.version=1.10.7
sbt.version=0.13.6

View File

@ -1,5 +1,12 @@
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2")
addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1")
addSbtPlugin("com.github.sbt" % "sbt-osgi" % "0.10.0")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4")
addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.1.0")
addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.7.1")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.2.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.3")
addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.3.0")
addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.5")
addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.7.0")

741
sbt
View File

@ -1,107 +1,87 @@
#!/usr/bin/env bash
#
# A more capable sbt runner, coincidentally also called sbt.
# Author: Paul Phillips <paulp@improving.org>
# https://github.com/paulp/sbt-extras
#
# Generated from http://www.opensource.org/licenses/bsd-license.php
# Copyright (c) 2011, Paul Phillips. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set -o pipefail
declare -r sbt_release_version="1.9.6"
declare -r sbt_unreleased_version="1.9.6"
declare -r latest_213="2.13.12"
declare -r latest_212="2.12.18"
declare -r latest_211="2.11.12"
declare -r latest_210="2.10.7"
declare -r latest_29="2.9.3"
declare -r latest_28="2.8.2"
# Author: Paul Phillips <paulp@typesafe.com>
# todo - make this dynamic
declare -r sbt_release_version="0.13.1"
declare -r sbt_unreleased_version="0.13.2-SNAPSHOT" # -sbt-dev doesn't work at present
declare -r buildProps="project/build.properties"
declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases"
declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots"
declare -r sbt_launch_mvn_release_repo="https://repo1.maven.org/maven2"
declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots"
declare sbt_jar sbt_dir sbt_create sbt_launch_dir
declare scala_version java_home sbt_explicit_version
declare verbose debug quiet noshare batch trace_level log_level
declare sbt_saved_stty
declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18"
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy -Dsbt.coursier.home=project/.coursier"
echoerr () { [[ -z "$quiet" ]] && echo "$@" >&2; }
vlog () { [[ -n "$verbose$debug" ]] && echoerr "$@"; }
dlog () { [[ -n "$debug" ]] && echoerr "$@"; }
declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new
declare sbt_explicit_version
declare verbose noshare batch trace_level
# we'd like these set before we get around to properly processing arguments
for arg in "$@"; do
case "$arg" in
-q|-quiet) quiet=true ;;
-d|-debug) debug=true ;;
-v|-verbose) verbose=true ;;
*) ;;
esac
done
declare java_cmd="java"
declare sbt_launch_dir="$HOME/.sbt/launchers"
declare sbt_launch_repo
# pull -J and -D options to give to java.
declare -a java_args scalac_args sbt_commands residual_args
# args to jvm/sbt via files or environment variables
declare -a extra_jvm_opts extra_sbt_opts
echoerr() { echo >&2 "$@"; }
vlog() { [[ -n "$verbose" ]] && echoerr "$@"; }
die() {
echo "Aborting: $*"
exit 1
# spaces are possible, e.g. sbt.version = 0.13.0
build_props_sbt () {
[[ -r "$buildProps" ]] && \
grep '^sbt\.version' "$buildProps" | tr '=' ' ' | awk '{ print $2; }'
}
setTrapExit() {
# save stty and trap exit, to ensure echo is re-enabled if we are interrupted.
SBT_STTY="$(stty -g 2>/dev/null)"
export SBT_STTY
update_build_props_sbt () {
local ver="$1"
local old="$(build_props_sbt)"
# restore stty settings (echo in particular)
onSbtRunnerExit() {
[ -t 0 ] || return
vlog ""
vlog "restoring stty: $SBT_STTY"
stty "$SBT_STTY"
[[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && {
perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
echoerr "!!!"
echoerr "!!! Updated file $buildProps setting sbt.version to: $ver"
echoerr "!!! Previous value was: $old"
echoerr "!!!"
}
vlog "saving stty: $SBT_STTY"
trap onSbtRunnerExit EXIT
}
sbt_version () {
if [[ -n "$sbt_explicit_version" ]]; then
echo "$sbt_explicit_version"
else
local v="$(build_props_sbt)"
if [[ -n "$v" ]]; then
echo "$v"
else
echo "$sbt_release_version"
fi
fi
}
# restore stty settings (echo in particular)
onSbtRunnerExit() {
[[ -n "$sbt_saved_stty" ]] || return
dlog ""
dlog "restoring stty: $sbt_saved_stty"
stty "$sbt_saved_stty"
unset sbt_saved_stty
}
# save stty and trap exit, to ensure echo is reenabled if we are interrupted.
trap onSbtRunnerExit EXIT
sbt_saved_stty="$(stty -g 2>/dev/null)"
dlog "Saved stty: $sbt_saved_stty"
# this seems to cover the bases on OSX, and someone will
# have to tell me about the others.
get_script_path() {
get_script_path () {
local path="$1"
[[ -L "$path" ]] || {
echo "$path"
return
}
[[ -L "$path" ]] || { echo "$path" ; return; }
local -r target="$(readlink "$path")"
local target="$(readlink "$path")"
if [[ "${target:0:1}" == "/" ]]; then
echo "$target"
else
@ -109,12 +89,23 @@ get_script_path() {
fi
}
script_path="$(get_script_path "${BASH_SOURCE[0]}")"
declare -r script_path
script_name="${script_path##*/}"
declare -r script_name
die() {
echo "Aborting: $@"
exit 1
}
init_default_option_file() {
make_url () {
version="$1"
case "$version" in
0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;;
0.10.* ) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
0.11.[12]) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
*) echo "$sbt_launch_repo/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
esac
}
init_default_option_file () {
local overriding_var="${!1}"
local default_file="$2"
if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then
@ -126,280 +117,112 @@ init_default_option_file() {
echo "$default_file"
}
sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
sbtx_opts_file="$(init_default_option_file SBTX_OPTS .sbtxopts)"
jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
declare -r default_jvm_opts="-Dfile.encoding=UTF8 -XX:MaxPermSize=384m -Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts"
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
declare -r latest_28="2.8.2"
declare -r latest_29="2.9.3"
declare -r latest_210="2.10.3"
declare -r latest_211="2.11.0-M5"
build_props_sbt() {
[[ -r "$buildProps" ]] &&
grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
}
declare -r script_path="$(get_script_path "$BASH_SOURCE")"
declare -r script_name="${script_path##*/}"
set_sbt_version() {
sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
[[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
export sbt_version
}
# some non-read-onlies set with defaults
declare java_cmd="java"
declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
declare sbt_launch_repo="http://typesafe.artifactoryonline.com/typesafe/ivy-releases"
url_base() {
local version="$1"
# pull -J and -D options to give to java.
declare -a residual_args
declare -a java_args
declare -a scalac_args
declare -a sbt_commands
case "$version" in
0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;;
0.10.*) echo "$sbt_launch_ivy_release_repo" ;;
0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;;
0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss"
echo "$sbt_launch_ivy_snapshot_repo" ;;
0.*) echo "$sbt_launch_ivy_release_repo" ;;
*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss"
echo "$sbt_launch_mvn_snapshot_repo" ;;
*) echo "$sbt_launch_mvn_release_repo" ;;
esac
}
# args to jvm/sbt via files or environment variables
declare -a extra_jvm_opts extra_sbt_opts
make_url() {
local version="$1"
# if set, use JAVA_HOME over java found in path
[[ -e "$JAVA_HOME/bin/java" ]] && java_cmd="$JAVA_HOME/bin/java"
local base="${sbt_launch_repo:-$(url_base "$version")}"
# directory to store sbt launchers
declare sbt_launch_dir="$HOME/.sbt/launchers"
[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
case "$version" in
0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;;
0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
*) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;;
esac
}
addJava() {
vlog "[addJava] arg = '$1'"
java_args+=("$1")
}
addSbt() {
vlog "[addSbt] arg = '$1'"
sbt_commands+=("$1")
}
addScalac() {
vlog "[addScalac] arg = '$1'"
scalac_args+=("$1")
}
addResidual() {
vlog "[residual] arg = '$1'"
residual_args+=("$1")
}
addResolver() { addSbt "set resolvers += $1"; }
addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; }
setThisBuild() {
vlog "[addBuild] args = '$*'"
local key="$1" && shift
addSbt "set $key in ThisBuild := $*"
}
setScalaVersion() {
[[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
addSbt "++ $1"
}
setJavaHome() {
java_cmd="$1/bin/java"
setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))"
export JAVA_HOME="$1"
export JDK_HOME="$1"
export PATH="$JAVA_HOME/bin:$PATH"
}
getJavaVersion() {
local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"')
# java -version on java8 says 1.8.x
# but on 9 and 10 it's 9.x.y and 10.x.y.
if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then
echo "${BASH_REMATCH[1]}"
# Fixes https://github.com/dwijnand/sbt-extras/issues/326
elif [[ "$str" =~ ^([0-9]+)(\..*)?(-ea)?$ ]]; then
echo "${BASH_REMATCH[1]}"
elif [[ -n "$str" ]]; then
echoerr "Can't parse java version from: $str"
build_props_scala () {
if [[ -r "$buildProps" ]]; then
versionLine="$(grep '^build.scala.versions' "$buildProps")"
versionString="${versionLine##build.scala.versions=}"
echo "${versionString%% .*}"
fi
}
checkJava() {
# Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME
[[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java"
[[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java"
if [[ -n "$java" ]]; then
pathJavaVersion=$(getJavaVersion java)
homeJavaVersion=$(getJavaVersion "$java")
if [[ "$pathJavaVersion" != "$homeJavaVersion" ]]; then
echoerr "Warning: Java version mismatch between PATH and JAVA_HOME/JDK_HOME, sbt will use the one in PATH"
echoerr " Either: fix your PATH, remove JAVA_HOME/JDK_HOME or use -java-home"
echoerr " java version from PATH: $pathJavaVersion"
echoerr " java version from JAVA_HOME/JDK_HOME: $homeJavaVersion"
fi
fi
}
java_version() {
local -r version=$(getJavaVersion "$java_cmd")
vlog "Detected Java version: $version"
echo "$version"
}
is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; }
# MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+
default_jvm_opts() {
local -r v="$(java_version)"
if [[ $v -ge 17 ]]; then
echo "$default_jvm_opts_common"
elif [[ $v -ge 10 ]]; then
if is_apple_silicon; then
# As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI
echo "$default_jvm_opts_common"
else
echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler"
fi
elif [[ $v -ge 8 ]]; then
echo "$default_jvm_opts_common"
else
echo "-XX:MaxPermSize=384m $default_jvm_opts_common"
fi
}
execRunner() {
execRunner () {
# print the arguments one to a line, quoting any containing spaces
vlog "# Executing command line:" && {
[[ "$verbose" || "$debug" ]] && echo "# Executing command line:" && {
for arg; do
if [[ -n "$arg" ]]; then
if printf "%s\n" "$arg" | grep -q ' '; then
printf >&2 "\"%s\"\n" "$arg"
printf "\"%s\"\n" "$arg"
else
printf >&2 "%s\n" "$arg"
printf "%s\n" "$arg"
fi
fi
done
vlog ""
echo ""
}
setTrapExit
if [[ -n "$batch" ]]; then
"$@" </dev/null
else
"$@"
exec </dev/null
fi
exec "$@"
}
jar_url() { make_url "$1"; }
is_cygwin() { [[ "$(uname -a)" == "CYGWIN"* ]]; }
jar_file() {
is_cygwin &&
cygpath -w "$sbt_launch_dir/$1/sbt-launch.jar" ||
echo "$sbt_launch_dir/$1/sbt-launch.jar"
jar_url () {
make_url "$1"
}
download_url() {
jar_file () {
echo "$sbt_launch_dir/$1/sbt-launch.jar"
}
download_url () {
local url="$1"
local jar="$2"
echo "Downloading sbt launcher for $(sbt_version):"
echo " From $url"
echo " To $jar"
mkdir -p "${jar%/*}" && {
if command -v curl >/dev/null 2>&1; then
curl --fail --silent --location "$url" --output "$jar"
elif command -v wget >/dev/null 2>&1; then
wget -q -O "$jar" "$url"
if which curl >/dev/null; then
curl --fail --silent "$url" --output "$jar"
elif which wget >/dev/null; then
wget --quiet -O "$jar" "$url"
fi
} && [[ -r "$jar" ]]
}
acquire_sbt_jar() {
{
sbt_jar="$(jar_file "$sbt_version")"
[[ -r "$sbt_jar" ]]
} || {
sbt_jar="$HOME/.ivy2/local/org.scala-sbt/sbt-launch/$sbt_version/jars/sbt-launch.jar"
[[ -r "$sbt_jar" ]]
} || {
sbt_jar="$(jar_file "$sbt_version")"
jar_url="$(make_url "$sbt_version")"
acquire_sbt_jar () {
for_sbt_version="$(sbt_version)"
sbt_url="$(jar_url "$for_sbt_version")"
sbt_jar="$(jar_file "$for_sbt_version")"
echoerr "Downloading sbt launcher for ${sbt_version}:"
echoerr " From ${jar_url}"
echoerr " To ${sbt_jar}"
download_url "${jar_url}" "${sbt_jar}"
case "${sbt_version}" in
0.*)
vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check"
echo ""
;;
*) verify_sbt_jar "${sbt_jar}" ;;
esac
}
[[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
}
verify_sbt_jar() {
local jar="${1}"
local md5="${jar}.md5"
md5url="$(make_url "${sbt_version}").md5"
echoerr "Downloading sbt launcher ${sbt_version} md5 hash:"
echoerr " From ${md5url}"
echoerr " To ${md5}"
download_url "${md5url}" "${md5}" >/dev/null 2>&1
if command -v md5sum >/dev/null 2>&1; then
if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then
rm -rf "${md5}"
return 0
else
echoerr "Checksum does not match"
return 1
fi
elif command -v md5 >/dev/null 2>&1; then
if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then
rm -rf "${md5}"
return 0
else
echoerr "Checksum does not match"
return 1
fi
elif command -v openssl >/dev/null 2>&1; then
if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then
rm -rf "${md5}"
return 0
else
echoerr "Checksum does not match"
return 1
fi
else
echoerr "Could not find an MD5 command"
return 1
fi
}
usage() {
set_sbt_version
usage () {
cat <<EOM
Usage: $script_name [options]
Note that options which are passed along to sbt begin with -- whereas
options to this runner use a single dash. Any sbt command can be scheduled
to run first by prefixing the command with --, so --warn, --error and so on
are not special.
-h | -help print this message
-v verbose operation (this runner is chattier)
-d, -w, -q aliases for --debug, --warn, --error (q means quiet)
-x debug this script
-v | -verbose this runner is chattier
-d | -debug set sbt log level to Debug
-q | -quiet set sbt log level to Error
-trace <level> display stack traces with a max of <level> frames (default: -1, traces suppressed)
-debug-inc enable debugging log for the incremental compiler
-no-colors disable ANSI color codes
-sbt-create start sbt even if current directory contains no sbt project
-sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt/<version>)
@ -410,58 +233,77 @@ are not special.
-jvm-debug <port> Turn on JVM debugging, open at the given port.
-batch Disable interactive mode
-prompt <expr> Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
-script <file> Run the specified file as a scala script
# sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
-sbt-version <version> use the specified version of sbt (default: $sbt_release_version)
-sbt-force-latest force the use of the latest release of sbt: $sbt_release_version
-sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version
-sbt-jar <path> use the specified jar as the sbt launcher
-sbt-launch-dir <path> directory to hold sbt launchers (default: $sbt_launch_dir)
-sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version"))
# sbt version (default: from $buildProps if present, else latest release)
!!! The only way to accomplish this pre-0.12.0 if there is a build.properties file which
!!! contains an sbt.version property is to update the file on disk. That's what this does.
-sbt-version <version> use the specified version of sbt (default: $sbt_release_version)
-sbt-jar <path> use the specified jar as the sbt launcher
-sbt-launch-dir <path> directory to hold sbt launchers (default: $sbt_launch_dir)
-sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $sbt_launch_repo)
# scala version (default: as chosen by sbt)
-28 use $latest_28
-29 use $latest_29
-210 use $latest_210
-211 use $latest_211
-212 use $latest_212
-213 use $latest_213
-scala-home <path> use the scala build at the specified directory
-scala-version <version> use the specified version of scala
-binary-version <version> use the specified scala version when searching for dependencies
-28 use $latest_28
-29 use $latest_29
-210 use $latest_210
-211 use $latest_211
-scala-home <path> use the scala build at the specified directory
-scala-version <version> use the specified version of scala
-binary-version <version> use the specified scala version when searching for dependencies
# java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
-java-home <path> alternate JAVA_HOME
-java-home <path> alternate JAVA_HOME
# passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
# The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
<default> $(default_jvm_opts)
JVM_OPTS environment variable holding either the jvm args directly, or
the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
-jvm-opts <path> file containing jvm args (if not given, .jvmopts in project root is used if present)
-Dkey=val pass -Dkey=val directly to the jvm
-J-X pass option -X directly to the jvm (-J is stripped)
<default> $default_jvm_opts
JVM_OPTS environment variable holding either the jvm args directly, or
the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
-jvm-opts <path> file containing jvm args (if not given, .jvmopts in project root is used if present)
-Dkey=val pass -Dkey=val directly to the jvm
-J-X pass option -X directly to the jvm (-J is stripped)
# passing options to sbt, OR to this runner
SBT_OPTS environment variable holding either the sbt args directly, or
the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
-sbt-opts <path> file containing sbt args (if not given, .sbtopts in project root is used if present)
-S-X add -X to sbt's scalacOptions (-S is stripped)
# passing options exclusively to this runner
SBTX_OPTS environment variable holding either the sbt-extras args directly, or
the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts')
Note: "@"-file is overridden by local '.sbtxopts' or '-sbtx-opts' argument.
-sbtx-opts <path> file containing sbt-extras args (if not given, .sbtxopts in project root is used if present)
SBT_OPTS environment variable holding either the sbt args directly, or
the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
-sbt-opts <path> file containing sbt args (if not given, .sbtopts in project root is used if present)
-S-X add -X to sbt's scalacOptions (-S is stripped)
EOM
exit 0
}
process_args() {
require_arg() {
addJava () {
dlog "[addJava] arg = '$1'"
java_args=( "${java_args[@]}" "$1" )
}
addSbt () {
dlog "[addSbt] arg = '$1'"
sbt_commands=( "${sbt_commands[@]}" "$1" )
}
addScalac () {
dlog "[addScalac] arg = '$1'"
scalac_args=( "${scalac_args[@]}" "$1" )
}
addResidual () {
dlog "[residual] arg = '$1'"
residual_args=( "${residual_args[@]}" "$1" )
}
addResolver () {
addSbt "set resolvers += $1"
}
addDebugger () {
addJava "-Xdebug"
addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
}
setScalaVersion () {
[[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
addSbt "++ $1"
}
process_args ()
{
require_arg () {
local type="$1"
local opt="$2"
local arg="$3"
@ -472,56 +314,45 @@ process_args() {
}
while [[ $# -gt 0 ]]; do
case "$1" in
-h | -help) usage ;;
-v) verbose=true && shift ;;
-d) addSbt "--debug" && shift ;;
-w) addSbt "--warn" && shift ;;
-q) addSbt "--error" && shift ;;
-x) shift ;; # currently unused
-trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
-h|-help) usage; exit 1 ;;
-v|-verbose) verbose=true && log_level=Info && shift ;;
-d|-debug) debug=true && log_level=Debug && shift ;;
-q|-quiet) quiet=true && log_level=Error && shift ;;
-no-colors) addJava "-Dsbt.log.noformat=true" && addJava "-Dsbt.color=false" && shift ;;
-sbt-create) sbt_create=true && shift ;;
-sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
-no-share) noshare=true && shift ;;
-offline) addSbt "set offline in Global := true" && shift ;;
-jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
-batch) batch=true && shift ;;
-prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
-script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;;
-trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
-no-share) noshare=true && shift ;;
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
-sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
-offline) addSbt "set offline := true" && shift ;;
-jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
-batch) batch=true && shift ;;
-prompt) require_arg "expr" "$1" "$2" && addSbt "set shellPrompt in ThisBuild := (s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
-sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
-sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;;
-sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;;
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
-sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
-sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
-sbt-create) sbt_create=true && shift ;;
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
-sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
-sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;;
-sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
-sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
-scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
-binary-version) require_arg version "$1" "$2" && addSbt "set scalaBinaryVersion in ThisBuild := \"$2\"" && shift 2 ;;
-scala-home) require_arg path "$1" "$2" && addSbt "set every scalaHome := Some(file(\"$2\"))" && shift 2 ;;
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
-sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
-jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
-28) setScalaVersion "$latest_28" && shift ;;
-29) setScalaVersion "$latest_29" && shift ;;
-210) setScalaVersion "$latest_210" && shift ;;
-211) setScalaVersion "$latest_211" && shift ;;
-212) setScalaVersion "$latest_212" && shift ;;
-213) setScalaVersion "$latest_213" && shift ;;
-D*) addJava "$1" && shift ;;
-J*) addJava "${1:2}" && shift ;;
-S*) addScalac "${1:2}" && shift ;;
-28) setScalaVersion "$latest_28" && shift ;;
-29) setScalaVersion "$latest_29" && shift ;;
-210) setScalaVersion "$latest_210" && shift ;;
-211) setScalaVersion "$latest_211" && shift ;;
-scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
-binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
-scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;;
-java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
-sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
-sbtx-opts) require_arg path "$1" "$2" && sbtx_opts_file="$2" && shift 2 ;;
-jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
-D*) addJava "$1" && shift ;;
-J*) addJava "${1:2}" && shift ;;
-S*) addScalac "${1:2}" && shift ;;
new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;;
*) addResidual "$1" && shift ;;
*) addResidual "$1" && shift ;;
esac
done
}
@ -531,74 +362,58 @@ process_args "$@"
# skip #-styled comments and blank lines
readConfigFile() {
local end=false
until $end; do
read -r || end=true
[[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY"
done <"$1"
while read line; do
[[ $line =~ ^# ]] || [[ -z $line ]] || echo "$line"
done < "$1"
}
# if there are file/environment sbt_opts, process again so we
# can supply args to this runner
if [[ -r "$sbt_opts_file" ]]; then
vlog "Using sbt options defined in file $sbt_opts_file"
while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file")
while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file")
elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then
vlog "Using sbt options defined in variable \$SBT_OPTS"
IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS"
extra_sbt_opts=( $SBT_OPTS )
else
vlog "No extra sbt options have been defined"
fi
# if there are file/environment sbtx_opts, process again so we
# can supply args to this runner
if [[ -r "$sbtx_opts_file" ]]; then
vlog "Using sbt options defined in file $sbtx_opts_file"
while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbtx_opts_file")
elif [[ -n "$SBTX_OPTS" && ! ("$SBTX_OPTS" =~ ^@.*) ]]; then
vlog "Using sbt options defined in variable \$SBTX_OPTS"
IFS=" " read -r -a extra_sbt_opts <<<"$SBTX_OPTS"
else
vlog "No extra sbt options have been defined"
fi
[[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}"
[[ -n "$extra_sbt_opts" ]] && process_args "${extra_sbt_opts[@]}"
# reset "$@" to the residual args
set -- "${residual_args[@]}"
argumentCount=$#
# set sbt version
set_sbt_version
checkJava
# only exists in 0.12+
setTraceLevel() {
case "$sbt_version" in
"0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;;
*) setThisBuild traceLevel "$trace_level" ;;
case "$(sbt_version)" in
"0.7."* | "0.10."* | "0.11."* )
echoerr "Cannot set trace level in sbt version $(sbt_version)"
;;
*)
addSbt "set every traceLevel := $trace_level"
;;
esac
}
# set scalacOptions if we were given any -S opts
[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[*]}\""
[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && addJava "-Dsbt.version=$sbt_explicit_version"
vlog "Detected sbt version $sbt_version"
# Update build.properties on disk to set explicit version - sbt gives us no choice
[[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
vlog "Detected sbt version $(sbt_version)"
if [[ -n "$sbt_script" ]]; then
residual_args=("$sbt_script" "${residual_args[@]}")
else
# no args - alert them there's stuff in here
((argumentCount > 0)) || {
vlog "Starting $script_name: invoke with -help for other options"
residual_args=(shell)
}
fi
[[ -n "$scala_version" ]] && echoerr "Overriding scala version to $scala_version"
# verify this is an sbt dir, -create was given or user attempts to run a scala script
[[ -r ./build.sbt || -d ./project || -n "$sbt_create" || -n "$sbt_script" || -n "$sbt_new" ]] || {
# no args - alert them there's stuff in here
(( argumentCount > 0 )) || {
vlog "Starting $script_name: invoke with -help for other options"
residual_args=( shell )
}
# verify this is an sbt dir or -create was given
[[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
cat <<EOM
$(pwd) doesn't appear to be an sbt project.
If you want to start sbt anyway, run:
@ -609,17 +424,12 @@ EOM
}
# pick up completion if present; todo
# shellcheck disable=SC1091
[[ -r .sbt_completion.sh ]] && source .sbt_completion.sh
# directory to store sbt launchers
[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
# no jar? download it.
[[ -r "$sbt_jar" ]] || acquire_sbt_jar || {
# still no jar? uh-oh.
echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar"
echo "Download failed. Obtain the jar manually and place it at $sbt_jar"
exit 1
}
@ -628,13 +438,13 @@ if [[ -n "$noshare" ]]; then
addJava "$opt"
done
else
case "$sbt_version" in
"0.7."* | "0.10."* | "0.11."* | "0.12."*)
case "$(sbt_version)" in
"0.7."* | "0.10."* | "0.11."* | "0.12."* )
[[ -n "$sbt_dir" ]] || {
sbt_dir="$HOME/.sbt/$sbt_version"
sbt_dir="$HOME/.sbt/$(sbt_version)"
vlog "Using $sbt_dir as sbt dir, -sbt-dir to override."
}
;;
;;
esac
if [[ -n "$sbt_dir" ]]; then
@ -644,18 +454,25 @@ fi
if [[ -r "$jvm_opts_file" ]]; then
vlog "Using jvm options defined in file $jvm_opts_file"
while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file")
while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file")
elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then
vlog "Using jvm options defined in \$JVM_OPTS variable"
IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS"
extra_jvm_opts=( $JVM_OPTS )
else
vlog "Using default jvm options"
IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)"
extra_jvm_opts=( $default_jvm_opts )
fi
# traceLevel is 0.12+
[[ -n "$trace_level" ]] && setTraceLevel
if [[ -n "$log_level" ]] && [[ "$log_level" != Info ]]; then
sbt_commands=("set logLevel in Global := Level.$log_level" "${sbt_commands[@]}")
fi
# run sbt
execRunner "$java_cmd" \
"${extra_jvm_opts[@]}" \
"${java_args[@]}" \

View File

@ -1,21 +0,0 @@
#!/bin/sh
# Compute sbt-dynver-compatible version number
BUILD_TIME=`date '+%Y%m%d-%H%M'`
DYN_VER=`git describe --long --tags --abbrev=8 --match "v[0-9]*" --always --dirty="-${BUILD_TIME}"`
GIT_DIST=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\2/g"`
GIT_TAG=`git describe --tags --dirty`
RELEASE_VERSION=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\1/g"`
SNAPSHOT_VERSION=`echo ${DYN_VER} | sed -re "s/v([^-]*)-([0-9]+)-g(.*)/\1-\2-\3/g"`-SNAPSHOT
if [ ${GIT_DIST} -eq 0 ]; then
if [ ${GIT_TAG} == *"-dirty" ]; then
VERSION=${SNAPSHOT_VERSION}
else
VERSION=${RELEASE_VERSION}
fi
else
VERSION=${SNAPSHOT_VERSION}
fi
echo ${VERSION}

View File

@ -1,15 +0,0 @@
#!/bin/bash
set -ev
VERSION=`perl -npe "s/version in ThisBuild\s+:=\s+\"(.*)\"/\1/" version.sbt | sed -e "/^$/d"`
# Deploy a snapshot version only for master branch and jdk8
if [[ "$TRAVIS_JDK_VERSION" == "oraclejdk8" ]]; then
if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] && [[ "$VERSION" == *SNAPSHOT ]]; then
./sbt "; test; publish";
else
./sbt test;
fi;
else
./sbt test;
fi;

View File

@ -1,117 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="FileTabCharacter"/>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\r"/>
<property name="message" value="Line contains carriage return"/>
</module>
<module name="RegexpMultiline">
<property name="format" value=" \n"/>
<property name="message" value="Line has trailing whitespace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\{\n\n"/>
<property name="message" value="Blank line after opening brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\}"/>
<property name="message" value="Blank line before closing brace"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\n"/>
<property name="message" value="Multiple consecutive blank lines"/>
</module>
<module name="RegexpMultiline">
<property name="format" value="\n\n\Z"/>
<property name="message" value="Blank line before end of file"/>
</module>
<module name="TreeWalker">
<module name="EmptyBlock">
<property name="option" value="text"/>
<property name="tokens" value="
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
</module>
<module name="EmptyStatement"/>
<module name="EmptyForInitializerPad"/>
<module name="EmptyForIteratorPad">
<property name="option" value="space"/>
</module>
<module name="MethodParamPad">
<property name="allowLineBreaks" value="true"/>
<property name="option" value="nospace"/>
</module>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
</module>
<module name="LeftCurly">
<property name="option" value="eol"/>
<property name="tokens" value="
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
</module>
<module name="RightCurly">
<property name="option" value="alone"/>
</module>
<module name="GenericWhitespace"/>
<module name="WhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="UpperEll"/>
<module name="DefaultComesLast"/>
<module name="ArrayTypeStyle"/>
<module name="MultipleVariableDeclarations"/>
<module name="ModifierOrder"/>
<module name="OneStatementPerLine"/>
<module name="StringLiteralEquality"/>
<module name="MutableException"/>
<module name="EqualsHashCode"/>
<module name="InnerAssignment"/>
<module name="InterfaceIsType"/>
<module name="HideUtilityClassConstructor"/>
<module name="MemberName"/>
<module name="LocalVariableName"/>
<module name="LocalFinalVariableName"/>
<module name="TypeName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="ClassTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="^[A-Z][0-9]?$"/>
</module>
<module name="AvoidStarImport"/>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="ignoreEnhancedForColon" value="false"/>
<property name="tokens" value="
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
</module>
</module>
</module>

View File

@ -1,280 +0,0 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// BitShuffle.java
// Since: 2016/03/31
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
public class BitShuffle
{
static {
try {
impl = SnappyLoader.loadBitShuffleApi();
}
catch (Exception e) {
throw new ExceptionInInitializerError(e);
}
}
/**
* An instance of BitShuffleNative
*/
private static BitShuffleNative impl;
/**
* Apply a bit-shuffling filter into the content in the given input buffer. After bit-shuffling,
* you can retrieve the shuffled data from the output buffer [pos() ...limit())
* (shuffled data size = limit() - pos() = remaining()).
*
* @param input buffer[pos() ... limit()) containing the input data
* @param type element type of the input data
* @param shuffled output of the shuffled data. Uses range [pos()..].
* @return byte size of the shuffled data.
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the input length is not a multiple of the given type size
*/
public static int shuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuffled) throws IOException {
if (!input.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = input.position();
int uLen = input.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("input length must be a multiple of the given type size: " + typeSize);
}
if (shuffled.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.shuffleDirectBuffer(input, uPos, typeSize, uLen, shuffled, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/**
* Apply a bit-shuffling filter into the input short array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] shuffle(short[] input) throws IOException {
if (input.length * 2 < input.length) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length);
}
byte[] output = new byte[input.length * 2];
int numProcessed = impl.shuffle(input, 0, 2, input.length * 2, output, 0);
assert(numProcessed == input.length * 2);
return output;
}
/**
* Apply a bit-shuffling filter into the input int array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] shuffle(int[] input) throws IOException {
if (input.length * 4 < input.length) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length);
}
byte[] output = new byte[input.length * 4];
int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0);
assert(numProcessed == input.length * 4);
return output;
}
/**
* Apply a bit-shuffling filter into the input long array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] shuffle(long[] input) throws IOException {
if (input.length * 8 < input.length) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length);
}
byte[] output = new byte[input.length * 8];
int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0);
assert(numProcessed == input.length * 8);
return output;
}
/**
* Apply a bit-shuffling filter into the input float array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] shuffle(float[] input) throws IOException {
if (input.length * 4 < input.length) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length);
}
byte[] output = new byte[input.length * 4];
int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0);
assert(numProcessed == input.length * 4);
return output;
}
/**
* Apply a bit-shuffling filter into the input double array.
*
* @param input
* @return bit-shuffled byte array
* @throws IOException
*/
public static byte[] shuffle(double[] input) throws IOException {
if (input.length * 8 < input.length) {
throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length);
}
byte[] output = new byte[input.length * 8];
int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0);
assert(numProcessed == input.length * 8);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original array. The result is dumped
* to the specified output buffer.
*
* @param shuffled buffer[pos() ... limit()) containing the input shuffled data
* @param type element type of the input data
* @param output output of the the original data. It uses buffer[pos()..]
* @return byte size of the unshuffled data.
* @throws IOException when failed to unshuffle the given input
* @throws SnappyError when the input is not a direct buffer
* @throws IllegalArgumentException when the length of input shuffled data is not a multiple of the given type size
*/
public static int unshuffle(ByteBuffer shuffled, BitShuffleType type, ByteBuffer output) throws IOException {
if (!shuffled.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "input is not a direct buffer");
}
if (!output.isDirect()) {
throw new SnappyError(SnappyErrorCode.NOT_A_DIRECT_BUFFER, "destination is not a direct buffer");
}
// input: input[pos(), limit())
// output: shuffled
int uPos = shuffled.position();
int uLen = shuffled.remaining();
int typeSize = type.getTypeSize();
if (uLen % typeSize != 0) {
throw new IllegalArgumentException("length of input shuffled data must be a multiple of the given type size: " + typeSize);
}
if (output.remaining() < uLen) {
throw new IllegalArgumentException("not enough space for output");
}
int numProcessed = impl.unshuffleDirectBuffer(shuffled, uPos, typeSize, uLen, output, shuffled.position());
assert(numProcessed == uLen);
// pos limit
// [ ......BBBBBBB.........]
shuffled.limit(shuffled.position() + numProcessed);
return numProcessed;
}
/**
* Convert the input bit-shuffled byte array into an original short array.
*
* @param input
* @return a short array
* @throws IOException
*/
public static short[] unshuffleShortArray(byte[] input) throws IOException {
short[] output = new short[input.length / 2];
int numProcessed = impl.unshuffle(input, 0, 2, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original int array.
*
* @param input
* @return an int array
* @throws IOException
*/
public static int[] unshuffleIntArray(byte[] input) throws IOException {
int[] output = new int[input.length / 4];
int numProcessed = impl.unshuffle(input, 0, 4, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original long array.
*
* @param input
* @return a long array
* @throws IOException
*/
public static long[] unshuffleLongArray(byte[] input) throws IOException {
long[] output = new long[input.length / 8];
int numProcessed = impl.unshuffle(input, 0, 8, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original float array.
*
* @param input
* @return an float array
* @throws IOException
*/
public static float[] unshuffleFloatArray(byte[] input) throws IOException {
float[] output = new float[input.length / 4];
int numProcessed = impl.unshuffle(input, 0, 4, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
/**
* Convert the input bit-shuffled byte array into an original double array.
*
* @param input
* @return a double array
* @throws IOException
*/
public static double[] unshuffleDoubleArray(byte[] input) throws IOException {
double[] output = new double[input.length / 8];
int numProcessed = impl.unshuffle(input, 0, 8, input.length, output, 0);
assert(numProcessed == input.length);
return output;
}
}

View File

@ -1,133 +0,0 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
#include <bitshuffle.h>
#include "BitShuffleNative.h"
inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
{
jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
if(c==0)
return;
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: shuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffle
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
int64_t processedBytes = bshuf_bitshuffle(
in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: shuffleDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffleDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: unshuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffle
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
int64_t processedBytes = bshuf_bitunshuffle(
in + inputOffset, out + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) processedBytes;
}
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: unshuffleDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffleDirectBuffer
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint typeSize, jint length, jobject output, jint outputOffset)
{
char* inputBuffer = (char*) env->GetDirectBufferAddress(input);
char* outputBuffer = (char*) env->GetDirectBufferAddress(output);
if(inputBuffer == 0 || outputBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
int64_t processedBytes = bshuf_bitunshuffle(
inputBuffer + inputOffset, outputBuffer + outputOffset, (size_t) (length / typeSize), (size_t) typeSize, 0);
return (jint) processedBytes;
}

View File

@ -1,45 +0,0 @@
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class org_xerial_snappy_BitShuffleNative */
#ifndef _Included_org_xerial_snappy_BitShuffleNative
#define _Included_org_xerial_snappy_BitShuffleNative
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: shuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: shuffleDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_shuffleDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: unshuffle
* Signature: (Ljava/lang/Object;IIILjava/lang/Object;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffle
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
/*
* Class: org_xerial_snappy_BitShuffleNative
* Method: unshuffleDirectBuffer
* Signature: (Ljava/nio/ByteBuffer;IIILjava/nio/ByteBuffer;I)I
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_BitShuffleNative_unshuffleDirectBuffer
(JNIEnv *, jobject, jobject, jint, jint, jint, jobject, jint);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -1,60 +0,0 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// BitShuffleNative.java
// Since: 2016/03/31
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* JNI interfaces of the {@link BitShuffle} implementation. The native method in this class is
* defined in BitShuffleNative.h (genereted by javah) and BitShuffleNative.cpp
* <p/>
* <p>
* <b> DO NOT USE THIS CLASS since the direct use of this class might break the
* native library code loading in {@link SnappyLoader}. </b>
* </p>
*
* @author leo
*/
public class BitShuffleNative
{
// ------------------------------------------------------------------------
// Bit-shuffling routines to improve compression of typed binary data.
// A quick benchmark result can be found in a gist below;
// https://gist.github.com/maropu/01103215df34b317a7a7
// ------------------------------------------------------------------------
public native int shuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
public native int shuffleDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
public native int unshuffle(Object input, int inputOffset, int typeSize, int byteLength, Object output, int outputOffset)
throws IOException;
public native int unshuffleDirectBuffer(ByteBuffer input, int inputOffset, int typeSize, int byteLength, ByteBuffer output, int outputOffset)
throws IOException;
}

View File

@ -1,53 +0,0 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyErrorCode.java
// Since: 2011/03/30 14:56:50
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Type codes used in ByteBuffer based BitShuffle APIs
*
* @author leo
*/
public enum BitShuffleType
{
BYTE(1),
SHORT(2),
INT(4),
LONG(8),
FLOAT(4),
DOUBLE(8);
public final int id;
private BitShuffleType(int id)
{
this.id = id;
}
public int getTypeSize()
{
return id;
}
}

View File

@ -24,18 +24,18 @@
//--------------------------------------
package org.xerial.snappy;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Locale;
/**
* Provides OS name and architecture name.
*
*
* @author leo
*
*/
public class OSInfo {
public class OSInfo
{
private static HashMap<String, String> archMapping = new HashMap<String, String>();
public static final String X86 = "x86";
@ -44,12 +44,6 @@ public class OSInfo {
public static final String IA64 = "ia64";
public static final String PPC = "ppc";
public static final String PPC64 = "ppc64";
public static final String PPC64LE = "ppc64le";
public static final String IBMZ = "s390";
public static final String IBMZ_64 = "s390x";
public static final String AARCH_64 = "aarch64";
public static final String RISCV_64 = "riscv64";
public static final String LOONGARCH_64 = "loongarch64";
static {
// x86 mappings
@ -81,39 +75,22 @@ public class OSInfo {
archMapping.put("power_pc", PPC);
archMapping.put("power_rs", PPC);
//PowerPC 64bit Little Endian mappings
archMapping.put(PPC64LE, PPC64LE);
// TODO: PowerPC 64bit mappings
archMapping.put(PPC64, PPC64);
archMapping.put("power64", PPC64);
archMapping.put("powerpc64", PPC64);
archMapping.put("power_pc64", PPC64);
archMapping.put("power_rs64", PPC64);
// IBM z mappings
archMapping.put(IBMZ, IBMZ);
// IBM z 64-bit mappings
archMapping.put(IBMZ_64, IBMZ_64);
// Aarch64 mappings
archMapping.put(AARCH_64, AARCH_64);
// RISC-V mappings
archMapping.put(RISCV_64, RISCV_64);
// LoongArch64 mappings
archMapping.put(LOONGARCH_64, LOONGARCH_64);
archMapping.put(PPC64, PPC64);
archMapping.put("power64", PPC64);
archMapping.put("powerpc64", PPC64);
archMapping.put("power_pc64", PPC64);
archMapping.put("power_rs64", PPC64);
}
public static void main(String[] args) {
if(args.length >= 1) {
if("--os".equals(args[0])) {
if (args.length >= 1) {
if ("--os".equals(args[0])) {
System.out.print(getOSName());
return;
}
else if("--arch".equals(args[0])) {
else if ("--arch".equals(args[0])) {
System.out.print(getArchName());
return;
}
@ -130,98 +107,27 @@ public class OSInfo {
return translateOSNameToFolderName(System.getProperty("os.name"));
}
public static boolean isAndroid() {
return System.getProperty("java.runtime.name", "").toLowerCase().contains("android");
}
static String getHardwareName() {
try {
Process p = Runtime.getRuntime().exec("uname -m");
p.waitFor();
InputStream in = p.getInputStream();
try {
int readLen = 0;
ByteArrayOutputStream b = new ByteArrayOutputStream();
byte[] buf = new byte[32];
while((readLen = in.read(buf, 0, buf.length)) >= 0) {
b.write(buf, 0, readLen);
}
return b.toString();
}
finally {
if(in != null) {
in.close();
}
}
}
catch(Throwable e) {
System.err.println("Error while running uname -m: " + e.getMessage());
return "unknown";
}
}
static String resolveArmArchType() {
if(System.getProperty("os.name").contains("Linux")) {
String armType = getHardwareName();
// armType (uname -m) can be armv5t, armv5te, armv5tej, armv5tejl, armv6, armv7, armv7l, i686
if(armType.startsWith("armv6")) {
// Raspberry PI
return "armv6";
}
else if(armType.startsWith("armv7")) {
// Generic
return "armv7";
}
// Java 1.8 introduces a system property to determine armel or armhf
// http://bugs.java.com/bugdatabase/view_bug.do?bug_id=8005545
String abi = System.getProperty("sun.arch.abi");
if(abi != null && abi.startsWith("gnueabihf")) {
return "armv7";
}
// For java7, we stil need to if run some shell commands to determine ABI of JVM
public static String getArchName() {
// if running Linux on ARM, need to determine ABI of JVM
String osArch = System.getProperty("os.arch");
if (osArch.startsWith("arm") && System.getProperty("os.name").contains("Linux")) {
String javaHome = System.getProperty("java.home");
try {
// determine if first JVM found uses ARM hard-float ABI
int exitCode = Runtime.getRuntime().exec("which readelf").waitFor();
if(exitCode == 0) {
String javaHome = System.getProperty("java.home");
String[] cmdarray = {"/bin/sh", "-c", "find '" + javaHome +
"' -name 'libjvm.so' | head -1 | xargs readelf -A | " +
"grep 'Tag_ABI_VFP_args: VFP registers'"};
exitCode = Runtime.getRuntime().exec(cmdarray).waitFor();
if(exitCode == 0) {
return "armv7";
}
}
else {
System.err.println("WARNING! readelf not found. Cannot check if running on an armhf system, " +
"armel architecture will be presumed.");
}
String[] cmdarray = { "/bin/sh", "-c", "find '" + javaHome +
"' -name 'libjvm.so' | head -1 | xargs readelf -A | " +
"grep 'Tag_ABI_VFP_args: VFP registers'" };
int exitCode = Runtime.getRuntime().exec(cmdarray).waitFor();
if (exitCode == 0)
return "armhf";
}
catch(IOException e) {
catch (IOException e) {
// ignored: fall back to "arm" arch (soft-float ABI)
}
catch(InterruptedException e) {
catch (InterruptedException e) {
// ignored: fall back to "arm" arch (soft-float ABI)
}
}
// Use armv5, soft-float ABI
return "arm";
}
public static String getArchName() {
String osArch = System.getProperty("os.arch");
// For Android
if(isAndroid()) {
return "android-arm";
}
if(osArch.startsWith("arm")) {
osArch = resolveArmArchType();
}
}
else {
String lc = osArch.toLowerCase(Locale.US);
if(archMapping.containsKey(lc))
@ -231,19 +137,20 @@ public class OSInfo {
}
static String translateOSNameToFolderName(String osName) {
if(osName.contains("Windows")) {
if (osName.contains("Windows")) {
return "Windows";
}
else if(osName.contains("Mac")) {
else if (osName.contains("Mac")) {
return "Mac";
}
else if(osName.contains("Linux")) {
else if (osName.contains("Linux")) {
return "Linux";
}
else if(osName.contains("AIX")) {
return "AIX";
}
else {
else if (osName.contains("AIX")) {
return "AIX";
}
else {
return osName.replaceAll("\\W", "");
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,60 +0,0 @@
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Snappy compressor/decompressor interface. The implementation can be JNI binding or pure-java Snappy implementation.
*/
public interface SnappyApi
{
// ------------------------------------------------------------------------
// Generic compression/decompression routines.
// ------------------------------------------------------------------------
long rawCompress(long inputAddr, long inputSize, long destAddr)
throws IOException;
long rawUncompress(long inputAddr, long inputSize, long destAddr)
throws IOException;
int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed,
int outputOffset)
throws IOException;
int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset)
throws IOException;
int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed,
int outputOffset)
throws IOException;
int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException;
// Returns the maximal size of the compressed representation of
// input data that is "source_bytes" bytes in length;
int maxCompressedLength(int source_bytes);
// This operation takes O(1) time.
int uncompressedLength(ByteBuffer compressed, int offset, int len)
throws IOException;
int uncompressedLength(Object input, int offset, int len)
throws IOException;
long uncompressedLength(long inputAddr, long len)
throws IOException;
boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len)
throws IOException;
boolean isValidCompressedBuffer(Object input, int offset, int len)
throws IOException;
boolean isValidCompressedBuffer(long inputAddr, long offset, long len)
throws IOException;
void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset)
throws IOException;
}

View File

@ -1,70 +1,67 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyBundleActivator.java
// Since: 2011/06/22 10:01:46
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleActivator;
import org.osgi.framework.BundleContext;
import java.util.jar.Manifest;
/**
* OSGi bundle entry point
*
* @author leo
*/
public class SnappyBundleActivator
implements BundleActivator
{
/**
* Name of the Snappy native library
*/
public static final String LIBRARY_NAME = "snappyjava";
/**
* Make a call to {@link System#loadLibrary(String)} to load the native library which assumes
* that the library is available on the path based on this {@link Bundle}'s {@link Manifest}.
*/
public void start(BundleContext context)
throws Exception
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyBundleActivator.java
// Since: 2011/06/22 10:01:46
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleActivator;
import org.osgi.framework.BundleContext;
import java.util.jar.Manifest;
/**
* OSGi bundle entry point
*
* @author leo
*
*/
public class SnappyBundleActivator implements BundleActivator
{
/**
* Name of the Snappy native library
*/
public static final String LIBRARY_NAME = "snappyjava";
/**
* Make a call to {@link System#loadLibrary(String)} to load the native library which assumes
* that the library is available on the path based on this {@link Bundle}'s {@link Manifest}.
*/
public void start(BundleContext context) throws Exception
{
String library = System.mapLibraryName(LIBRARY_NAME);
String osArch = System.getProperty("os.arch");
if (library.toLowerCase().endsWith(".dylib") && "x86".equals(osArch)) {
// some MacOS JDK7+ vendors map to dylib instead of jnilib
library = library.replace(".dylib", ".jnilib");
}
System.loadLibrary(library);
SnappyLoader.setSnappyApi(new SnappyNative());
}
public void stop(BundleContext context)
throws Exception
{
SnappyLoader.setSnappyApi(null);
SnappyLoader.cleanUpExtractedNativeLib();
}
}
String library = System.mapLibraryName(LIBRARY_NAME);
if (library.toLowerCase().endsWith(".dylib"))
{
// some MacOS JDK7+ vendors map to dylib instead of jnilib
library = library.replace(".dylib", ".jnilib");
}
System.loadLibrary(library);
SnappyLoader.setApi(new SnappyNative());
}
public void stop(BundleContext context) throws Exception
{
SnappyLoader.setApi(null);
SnappyLoader.cleanUpExtractedNativeLib();
}
}

View File

@ -1,146 +1,117 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyCodec.java
// Since: 2011/04/03 14:50:20
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
/**
* Preamble header for {@link SnappyOutputStream}.
* <p/>
* <p>
* The magic header is the following 8 bytes data:
* <p/>
* <pre>
* -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0
* </pre>
* <p/>
* </p>
*
* @author leo
*/
public class SnappyCodec
{
static final byte[] MAGIC_HEADER = new byte[] {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0};
public static final int MAGIC_LEN = MAGIC_HEADER.length;
public static final int HEADER_SIZE = MAGIC_LEN + 8;
public static final int MAGIC_HEADER_HEAD = SnappyOutputStream.readInt(MAGIC_HEADER, 0);
static {
assert (MAGIC_HEADER_HEAD < 0);
}
public static final int DEFAULT_VERSION = 1;
public static final int MINIMUM_COMPATIBLE_VERSION = 1;
public static final SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
public final byte[] magic;
public final int version;
public final int compatibleVersion;
private final byte[] headerArray;
private SnappyCodec(byte[] magic, int version, int compatibleVersion)
{
this.magic = magic;
this.version = version;
this.compatibleVersion = compatibleVersion;
ByteArrayOutputStream header = new ByteArrayOutputStream(HEADER_SIZE);
DataOutputStream d = new DataOutputStream(header);
try {
d.write(magic, 0, MAGIC_LEN);
d.writeInt(version);
d.writeInt(compatibleVersion);
d.close();
}
catch (IOException e) {
throw new RuntimeException(e);
}
headerArray = header.toByteArray();
}
public static byte[] getMagicHeader()
{
return MAGIC_HEADER.clone();
}
@Override
public String toString()
{
return String.format("version:%d, compatible version:%d", version, compatibleVersion);
}
public static int headerSize()
{
return HEADER_SIZE;
}
public int writeHeader(byte[] dst, int dstOffset)
{
System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length);
return headerArray.length;
}
public int writeHeader(OutputStream out)
throws IOException
{
out.write(headerArray, 0, headerArray.length);
return headerArray.length;
}
public boolean isValidMagicHeader()
{
return Arrays.equals(MAGIC_HEADER, magic);
}
public static boolean hasMagicHeaderPrefix(byte[] b) {
int limit = Math.min(MAGIC_LEN, b.length);
int i = 0;
while(i < limit) {
if(b[i] != MAGIC_HEADER[i]) {
return false;
}
++i;
}
return true;
}
public static SnappyCodec readHeader(InputStream in)
throws IOException
{
DataInputStream d = new DataInputStream(in);
byte[] magic = new byte[MAGIC_LEN];
d.readFully(magic, 0, MAGIC_LEN);
int version = d.readInt();
int compatibleVersion = d.readInt();
return new SnappyCodec(magic, version, compatibleVersion);
}
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyCodec.java
// Since: 2011/04/03 14:50:20
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
/**
* Preamble header for {@link SnappyOutputStream}.
*
* <p>
* The magic header is the following 8 bytes data:
*
* <pre>
* -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0
* </pre>
*
* </p>
*
* @author leo
*
*/
public class SnappyCodec
{
public static final byte[] MAGIC_HEADER = new byte[] { -126, 'S', 'N', 'A', 'P', 'P', 'Y', 0 };
public static final int MAGIC_LEN = MAGIC_HEADER.length;
public static final int HEADER_SIZE = MAGIC_LEN + 8;
public static final int DEFAULT_VERSION = 1;
public static final int MINIMUM_COMPATIBLE_VERSION = 1;
public final byte[] magic;
public final int version;
public final int compatibleVersion;
private final byte[] headerArray;
private SnappyCodec(byte[] magic, int version, int compatibleVersion) {
this.magic = magic;
this.version = version;
this.compatibleVersion = compatibleVersion;
ByteArrayOutputStream header = new ByteArrayOutputStream(HEADER_SIZE);
DataOutputStream d = new DataOutputStream(header);
try {
d.write(magic, 0, MAGIC_LEN);
d.writeInt(version);
d.writeInt(compatibleVersion);
d.close();
}
catch(IOException e) {
throw new RuntimeException(e);
}
headerArray = header.toByteArray();
}
@Override
public String toString() {
return String.format("version:%d, compatible version:%d", version, compatibleVersion);
}
public static int headerSize() {
return HEADER_SIZE;
}
public int writeHeader(byte[] dst, int dstOffset) {
System.arraycopy(headerArray, 0, dst, dstOffset, headerArray.length);
return headerArray.length;
}
public int writeHeader(OutputStream out) throws IOException {
out.write(headerArray, 0, headerArray.length);
return headerArray.length;
}
public boolean isValidMagicHeader() {
return Arrays.equals(MAGIC_HEADER, magic);
}
public static SnappyCodec readHeader(InputStream in) throws IOException {
DataInputStream d = new DataInputStream(in);
byte[] magic = new byte[MAGIC_LEN];
d.readFully(magic, 0, MAGIC_LEN);
int version = d.readInt();
int compatibleVersion = d.readInt();
return new SnappyCodec(magic, version, compatibleVersion);
}
public static SnappyCodec currentHeader = new SnappyCodec(MAGIC_HEADER, DEFAULT_VERSION, MINIMUM_COMPATIBLE_VERSION);
}

View File

@ -1,65 +1,62 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyError.java
// Since: 2011/03/30 15:22:43
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Used when serious errors (unchecked exception) are observed.
*
* @author leo
*/
public class SnappyError
extends Error
{
/**
*
*/
private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode;
public SnappyError(SnappyErrorCode code)
{
super();
this.errorCode = code;
}
public SnappyError(SnappyErrorCode code, Error e)
{
super(e);
this.errorCode = code;
}
public SnappyError(SnappyErrorCode code, String message)
{
super(message);
this.errorCode = code;
}
@Override
public String getMessage()
{
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyError.java
// Since: 2011/03/30 15:22:43
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Used when serious errors (unchecked exception) are observed.
*
* @author leo
*
*/
public class SnappyError extends Error
{
/**
*
*/
private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode;
public SnappyError(SnappyErrorCode code) {
super();
this.errorCode = code;
}
public SnappyError(SnappyErrorCode code, Error e) {
super(e);
this.errorCode = code;
}
public SnappyError(SnappyErrorCode code, String message) {
super(message);
this.errorCode = code;
}
@Override
public String getMessage() {
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
}

View File

@ -1,69 +1,64 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyErrorCode.java
// Since: 2011/03/30 14:56:50
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Error codes of snappy-java
*
* @author leo
*/
public enum SnappyErrorCode
{
// DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp
UNKNOWN(0),
FAILED_TO_LOAD_NATIVE_LIBRARY(1),
PARSING_ERROR(2),
NOT_A_DIRECT_BUFFER(3),
OUT_OF_MEMORY(4),
FAILED_TO_UNCOMPRESS(5),
EMPTY_INPUT(6),
INCOMPATIBLE_VERSION(7),
INVALID_CHUNK_SIZE(8),
UNSUPPORTED_PLATFORM(9),
TOO_LARGE_INPUT(10);
public final int id;
private SnappyErrorCode(int id)
{
this.id = id;
}
public static SnappyErrorCode getErrorCode(int id)
{
for (SnappyErrorCode code : SnappyErrorCode.values()) {
if (code.id == id) {
return code;
}
}
return UNKNOWN;
}
public static String getErrorMessage(int id)
{
return getErrorCode(id).name();
}
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyErrorCode.java
// Since: 2011/03/30 14:56:50
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
/**
* Error codes of snappy-java
*
* @author leo
*
*/
public enum SnappyErrorCode {
// DO NOT change these error code IDs because these numbers are used inside SnappyNative.cpp
UNKNOWN(0),
FAILED_TO_LOAD_NATIVE_LIBRARY(1),
PARSING_ERROR(2),
NOT_A_DIRECT_BUFFER(3),
OUT_OF_MEMORY(4),
FAILED_TO_UNCOMPRESS(5),
EMPTY_INPUT(6),
INCOMPATIBLE_VERSION(7),
INVALID_CHUNK_SIZE(8)
;
public final int id;
private SnappyErrorCode(int id) {
this.id = id;
}
public static SnappyErrorCode getErrorCode(int id) {
for (SnappyErrorCode code : SnappyErrorCode.values()) {
if (code.id == id)
return code;
}
return UNKNOWN;
}
public static String getErrorMessage(int id) {
return getErrorCode(id).name();
}
}

View File

@ -1,82 +1,74 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyException.java
// Since: 2011/03/30 14:56:14
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
/**
* Exception in snappy-java
*
* @author leo
* @deprecated Snappy-java now uses {@link IOException}
*/
@Deprecated
public class SnappyException
extends Exception
{
private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode;
public SnappyException(int code)
{
this(SnappyErrorCode.getErrorCode(code));
}
public SnappyException(SnappyErrorCode errorCode)
{
super();
this.errorCode = errorCode;
}
public SnappyException(SnappyErrorCode errorCode, Exception e)
{
super(e);
this.errorCode = errorCode;
}
public SnappyException(SnappyErrorCode errorCode, String message)
{
super(message);
this.errorCode = errorCode;
}
public SnappyErrorCode getErrorCode()
{
return errorCode;
}
public static void throwException(int errorCode)
throws SnappyException
{
throw new SnappyException(errorCode);
}
@Override
public String getMessage()
{
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// XerialJ
//
// SnappyException.java
// Since: 2011/03/30 14:56:14
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
/**
* Exception in snappy-java
*
* @deprecated Snappy-java now uses {@link IOException}
* @author leo
*
*/
@Deprecated
public class SnappyException extends Exception
{
private static final long serialVersionUID = 1L;
public final SnappyErrorCode errorCode;
public SnappyException(int code) {
this(SnappyErrorCode.getErrorCode(code));
}
public SnappyException(SnappyErrorCode errorCode) {
super();
this.errorCode = errorCode;
}
public SnappyException(SnappyErrorCode errorCode, Exception e) {
super(e);
this.errorCode = errorCode;
}
public SnappyException(SnappyErrorCode errorCode, String message) {
super(message);
this.errorCode = errorCode;
}
public SnappyErrorCode getErrorCode() {
return errorCode;
}
public static void throwException(int errorCode) throws SnappyException {
throw new SnappyException(errorCode);
}
@Override
public String getMessage() {
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
}

View File

@ -4,70 +4,52 @@
package org.xerial.snappy;
import java.io.IOException;
import java.lang.invoke.LambdaMetafactory;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.Checksum;
/**
* Constants and utilities for implementing x-snappy-framed.
*
*
* @author Brett Okken
* @since 1.1.0
*/
final class SnappyFramed
{
final class SnappyFramed {
public static final int COMPRESSED_DATA_FLAG = 0x00;
public static final int UNCOMPRESSED_DATA_FLAG = 0x01;
public static final int STREAM_IDENTIFIER_FLAG = 0xff;
private static final int MASK_DELTA = 0xa282ead8;
private static final Supplier<Checksum> CHECKSUM_SUPPLIER;
/**
* Sun specific mechanisms to clean up resources associated with direct byte buffers.
*/
@SuppressWarnings("unchecked")
private static final Class<? extends ByteBuffer> SUN_DIRECT_BUFFER = (Class<? extends ByteBuffer>) lookupClassQuietly("sun.nio.ch.DirectBuffer");
private static final Method SUN_BUFFER_CLEANER;
private static final Method SUN_CLEANER_CLEAN;
static
{
Supplier<Checksum> supplier = null;
try
{
final Class<?> crc32cClazz = Class.forName("java.util.zip.CRC32C");
// using LambdaMetafactory requires a caller sensitive lookup
final MethodHandles.Lookup lookup = MethodHandles.lookup();
final MethodHandle conHandle = lookup.findConstructor(crc32cClazz, MethodType.methodType(void.class));
// use LambdaMetafactory to generate an implementation of Supplier<Checksum> which invokes
// the java.util.zip.CRC32C default constructor
supplier = (Supplier<Checksum>) LambdaMetafactory.metafactory(lookup,
// method name on Supplier
"get",
// functional interface to be created by factory
MethodType.methodType(Supplier.class),
// type of the functional interface
// uses a generic, so erasure to Object
MethodType.methodType(Object.class),
// the method handle to call
conHandle,
// type as used at call site
MethodType.methodType(Checksum.class))
.getTarget()
.invoke();
Method bufferCleaner = null;
Method cleanerClean = null;
try {
//operate under the assumption that if the sun direct buffer class exists,
//all of the sun classes exist
if (SUN_DIRECT_BUFFER != null) {
bufferCleaner = SUN_DIRECT_BUFFER.getMethod("cleaner", (Class[])null);
Class<?> cleanClazz = lookupClassQuietly("sun.misc.Cleaner");
cleanerClean = cleanClazz.getMethod("clean", (Class[])null);
}
} catch(Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t);
}
catch(Throwable t)
{
Logger.getLogger(SnappyFramed.class.getName())
.log(Level.FINE, "java.util.zip.CRC32C not loaded, using PureJavaCrc32C", t);
supplier = null;
}
CHECKSUM_SUPPLIER = supplier != null ? supplier : PureJavaCrc32C::new;
SUN_BUFFER_CLEANER = bufferCleaner;
SUN_CLEANER_CLEAN = cleanerClean;
}
/**
@ -76,18 +58,18 @@ final class SnappyFramed
*/
public static final byte[] HEADER_BYTES = new byte[] {
(byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
0x50, 0x70, 0x59};
0x50, 0x70, 0x59 };
public static Checksum getCRC32C()
public static int maskedCrc32c(byte[] data)
{
return CHECKSUM_SUPPLIER.get();
return maskedCrc32c(data, 0, data.length);
}
public static int maskedCrc32c(Checksum crc32c, byte[] data, int offset, int length)
public static int maskedCrc32c(byte[] data, int offset, int length)
{
crc32c.reset();
final PureJavaCrc32C crc32c = new PureJavaCrc32C();
crc32c.update(data, offset, length);
return mask((int) crc32c.getValue());
return mask(crc32c.getIntegerValue());
}
/**
@ -96,10 +78,10 @@ final class SnappyFramed
* in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
* 0xa282ead8 (using wraparound as normal for unsigned integers). This is
* equivalent to the following C code:
* <p/>
*
* <pre>
* uint32_t mask_checksum(uint32_t x) {
* return ((x >> 15) | (x << 17)) + 0xa282ead8;
* return ((x >> 15) | (x << 17)) + 0xa282ead8;
* }
* </pre>
*/
@ -108,9 +90,9 @@ final class SnappyFramed
// Rotate right by 15 bits and add a constant.
return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
}
static final int readBytes(ReadableByteChannel source, ByteBuffer dest)
throws IOException
static final int readBytes(ReadableByteChannel source, ByteBuffer dest) throws IOException
{
// tells how many bytes to read.
final int expectedLength = dest.remaining();
@ -123,43 +105,47 @@ final class SnappyFramed
totalRead = lastRead;
// if we did not read as many bytes as we had hoped, try reading again.
if (lastRead < expectedLength) {
if (lastRead < expectedLength)
{
// as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
while (dest.remaining() != 0 && lastRead != -1) {
while (dest.remaining() != 0 && lastRead != -1)
{
lastRead = source.read(dest);
// if we got EOF, do not add to total read.
if (lastRead != -1) {
if (lastRead != -1)
{
totalRead += lastRead;
}
}
}
if (totalRead > 0) {
if (totalRead > 0)
{
dest.limit(dest.position());
}
else {
else
{
dest.position(dest.limit());
}
return totalRead;
}
static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer)
throws IOException
static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer) throws IOException
{
if (skip <= 0) {
return 0;
}
int toSkip = skip;
int skipped = 0;
while (toSkip > 0 && skipped != -1) {
int skipped = 0;
while(toSkip > 0 && skipped != -1) {
buffer.clear();
if (toSkip < buffer.capacity()) {
buffer.limit(toSkip);
}
skipped = source.read(buffer);
if (skipped > 0) {
toSkip -= skipped;
@ -169,4 +155,32 @@ final class SnappyFramed
buffer.clear();
return skip - toSkip;
}
private static Class<?> lookupClassQuietly(String name) {
try {
return SnappyFramed.class.getClassLoader().loadClass(name);
} catch (Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t);
}
return null;
}
/**
* Provides jvm implementation specific operation to aggressively release resources associated with <i>buffer</i>.
* @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}.
*/
static void releaseDirectByteBuffer(ByteBuffer buffer)
{
assert buffer != null && buffer.isDirect();
if (SUN_DIRECT_BUFFER != null && SUN_DIRECT_BUFFER.isAssignableFrom(buffer.getClass())) {
try {
Object cleaner = SUN_BUFFER_CLEANER.invoke(buffer, (Object[]) null);
SUN_CLEANER_CLEAN.invoke(cleaner, (Object[]) null);
} catch (Throwable t) {
Logger.getLogger(SnappyFramed.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t);
}
}
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,30 +0,0 @@
package org.xerial.snappy;
import org.xerial.snappy.buffer.CachedBufferAllocator;
import java.io.OutputStream;
public class SnappyHadoopCompatibleOutputStream extends SnappyOutputStream
{
public SnappyHadoopCompatibleOutputStream(OutputStream out)
{
this(out, DEFAULT_BLOCK_SIZE);
}
public SnappyHadoopCompatibleOutputStream(OutputStream out, int blockSize)
{
super(out, blockSize, CachedBufferAllocator.getBufferAllocatorFactory());
}
@Override
protected int writeHeader()
{
return 0;
}
@Override
protected void writeBlockPreemble()
{
writeCurrentDataSize();
}
}

View File

@ -5,22 +5,19 @@ import java.io.IOException;
/**
* Enhanced IOException with SnappyErrorCode
*/
public class SnappyIOException
extends IOException
{
public class SnappyIOException extends IOException {
private final SnappyErrorCode errorCode;
public SnappyIOException(SnappyErrorCode errorCode, String message)
{
public SnappyIOException(SnappyErrorCode errorCode, String message) {
super(message);
this.errorCode = errorCode;
}
@Override
public String getMessage()
{
public String getMessage() {
return String.format("[%s] %s", errorCode.name(), super.getMessage());
}
public SnappyErrorCode getErrorCode() { return errorCode; }
}

View File

@ -30,49 +30,30 @@ import java.io.InputStream;
/**
* A stream filter for reading data compressed by {@link SnappyOutputStream}.
*
*
*
* @author leo
*
*/
public class SnappyInputStream
extends InputStream
public class SnappyInputStream extends InputStream
{
public static final int MAX_CHUNK_SIZE = 512 * 1024 * 1024; // 512 MiB
private boolean finishedReading = false;
private boolean finishedReading = false;
protected final InputStream in;
private final int maxChunkSize;
private byte[] compressed;
private byte[] uncompressed;
private int uncompressedCursor = 0;
private int uncompressedLimit = 0;
private byte[] compressed;
private byte[] uncompressed;
private int uncompressedCursor = 0;
private int uncompressedLimit = 0;
private byte[] header = new byte[SnappyCodec.headerSize()];
private byte[] chunkSizeBuf = new byte[4];
/**
* Create a filter for reading compressed data as a uncompressed stream
*
*
* @param input
* @throws IOException
*/
public SnappyInputStream(InputStream input)
throws IOException
{
this(input, MAX_CHUNK_SIZE);
}
/**
* Create a filter for reading compressed data as a uncompressed stream with provided maximum chunk size
*
* @param input
* @param maxChunkSize
* @throws IOException
*/
public SnappyInputStream(InputStream input, int maxChunkSize)
throws IOException
{
this.maxChunkSize = maxChunkSize;
public SnappyInputStream(InputStream input) throws IOException {
this.in = input;
readHeader();
}
@ -84,44 +65,34 @@ public class SnappyInputStream
* @see java.io.InputStream#close()
*/
@Override
public void close()
throws IOException
{
public void close() throws IOException {
compressed = null;
uncompressed = null;
if (in != null) {
if (in != null)
in.close();
}
}
protected void readHeader()
throws IOException
{
protected void readHeader() throws IOException {
byte[] header = new byte[SnappyCodec.headerSize()];
int readBytes = 0;
while (readBytes < header.length) {
int ret = in.read(header, readBytes, header.length - readBytes);
if (ret == -1) {
if (ret == -1)
break;
}
readBytes += ret;
}
// Quick test of the header
if (readBytes == 0) {
if(readBytes == 0) {
// Snappy produces at least 1-byte result. So the empty input is not a valid input
throw new SnappyIOException(SnappyErrorCode.EMPTY_INPUT, "Cannot decompress empty stream");
}
if (readBytes < header.length || !SnappyCodec.hasMagicHeaderPrefix(header)) {
if (readBytes < header.length || header[0] != SnappyCodec.MAGIC_HEADER[0]) {
// do the default uncompression
// (probably) compressed by Snappy.compress(byte[])
readFully(header, readBytes);
return;
}
}
private static boolean isValidHeader(byte[] header)
throws IOException
{
SnappyCodec codec = SnappyCodec.readHeader(new ByteArrayInputStream(header));
if (codec.isValidMagicHeader()) {
// The input data is compressed by SnappyOutputStream
@ -130,17 +101,16 @@ public class SnappyInputStream
"Compressed with an incompatible codec version %d. At least version %d is required",
codec.version, SnappyCodec.MINIMUM_COMPATIBLE_VERSION));
}
return true;
}
else {
return false;
// (probably) compressed by Snappy.compress(byte[])
readFully(header, readBytes);
return;
}
}
protected void readFully(byte[] fragment, int fragmentLength)
throws IOException
{
if (fragmentLength == 0) {
protected void readFully(byte[] fragment, int fragmentLength) throws IOException {
if(fragmentLength == 0) {
finishedReading = true;
return;
}
@ -148,7 +118,7 @@ public class SnappyInputStream
compressed = new byte[Math.max(8 * 1024, fragmentLength)]; // 8K
System.arraycopy(fragment, 0, compressed, 0, fragmentLength);
int cursor = fragmentLength;
for (int readBytes = 0; (readBytes = in.read(compressed, cursor, compressed.length - cursor)) != -1; ) {
for (int readBytes = 0; (readBytes = in.read(compressed, cursor, compressed.length - cursor)) != -1;) {
cursor += readBytes;
if (cursor >= compressed.length) {
byte[] newBuf = new byte[(compressed.length * 2)];
@ -165,6 +135,7 @@ public class SnappyInputStream
Snappy.uncompress(compressed, 0, cursor, uncompressed, 0);
this.uncompressedCursor = 0;
this.uncompressedLimit = uncompressedLength;
}
/**
@ -175,48 +146,26 @@ public class SnappyInputStream
* @see java.io.InputStream#read(byte[], int, int)
*/
@Override
public int read(byte[] b, int byteOffset, int byteLength)
throws IOException
{
int writtenBytes = 0;
for (; writtenBytes < byteLength; ) {
if (uncompressedCursor >= uncompressedLimit) {
if (hasNextChunk()) {
continue;
}
else {
return writtenBytes == 0 ? -1 : writtenBytes;
}
}
int bytesToWrite = Math.min(uncompressedLimit - uncompressedCursor, byteLength - writtenBytes);
System.arraycopy(uncompressed, uncompressedCursor, b, byteOffset + writtenBytes, bytesToWrite);
writtenBytes += bytesToWrite;
uncompressedCursor += bytesToWrite;
}
return writtenBytes;
public int read(byte[] b, int off, int len) throws IOException {
return rawRead(b, off, len);
}
/**
* Read uncompressed data into the specified array
*
*
* @param array
* @param byteOffset
* @param byteLength
* @return written bytes
* @throws IOException
*/
public int rawRead(Object array, int byteOffset, int byteLength)
throws IOException
{
public int rawRead(Object array, int byteOffset, int byteLength) throws IOException {
int writtenBytes = 0;
for (; writtenBytes < byteLength; ) {
for (; writtenBytes < byteLength;) {
if (uncompressedCursor >= uncompressedLimit) {
if (hasNextChunk()) {
if (hasNextChunk())
continue;
}
else {
return writtenBytes == 0 ? -1 : writtenBytes;
}
@ -232,235 +181,175 @@ public class SnappyInputStream
/**
* Read long array from the stream
*
* @param d input
* @param off offset
* @param len the number of long elements to read
*
* @param d
* input
* @param off
* offset
* @param len
* the number of long elements to read
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(long[] d, int off, int len)
throws IOException
{
public int read(long[] d, int off, int len) throws IOException {
return rawRead(d, off * 8, len * 8);
}
/**
* Read long array from the stream
*
*
* @param d
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(long[] d)
throws IOException
{
public int read(long[] d) throws IOException {
return read(d, 0, d.length);
}
/**
* Read double array from the stream
*
* @param d input
* @param off offset
* @param len the number of double elements to read
*
* @param d
* input
* @param off
* offset
* @param len
* the number of double elements to read
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(double[] d, int off, int len)
throws IOException
{
public int read(double[] d, int off, int len) throws IOException {
return rawRead(d, off * 8, len * 8);
}
/**
* Read double array from the stream
*
*
* @param d
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(double[] d)
throws IOException
{
public int read(double[] d) throws IOException {
return read(d, 0, d.length);
}
/**
* Read int array from the stream
*
*
* @param d
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(int[] d)
throws IOException
{
public int read(int[] d) throws IOException {
return read(d, 0, d.length);
}
/**
* Read int array from the stream
*
* @param d input
* @param off offset
* @param len the number of int elements to read
*
* @param d
* input
* @param off
* offset
* @param len
* the number of int elements to read
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(int[] d, int off, int len)
throws IOException
{
public int read(int[] d, int off, int len) throws IOException {
return rawRead(d, off * 4, len * 4);
}
/**
* Read float array from the stream
*
* @param d input
* @param off offset
* @param len the number of float elements to read
*
* @param d
* input
* @param off
* offset
* @param len
* the number of float elements to read
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(float[] d, int off, int len)
throws IOException
{
public int read(float[] d, int off, int len) throws IOException {
return rawRead(d, off * 4, len * 4);
}
/**
* Read float array from the stream
*
*
* @param d
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(float[] d)
throws IOException
{
public int read(float[] d) throws IOException {
return read(d, 0, d.length);
}
/**
* Read short array from the stream
*
* @param d input
* @param off offset
* @param len the number of short elements to read
*
* @param d
* input
* @param off
* offset
* @param len
* the number of short elements to read
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(short[] d, int off, int len)
throws IOException
{
public int read(short[] d, int off, int len) throws IOException {
return rawRead(d, off * 2, len * 2);
}
/**
* Read short array from the stream
*
*
* @param d
* @return the total number of bytes read into the buffer, or -1 if there is
* no more data because the end of the stream has been reached.
* no more data because the end of the stream has been reached.
* @throws IOException
*/
public int read(short[] d)
throws IOException
{
public int read(short[] d) throws IOException {
return read(d, 0, d.length);
}
/**
* Read next len bytes
*
* @param dest
* @param offset
* @param len
* @return read bytes
*/
private int readNext(byte[] dest, int offset, int len)
throws IOException
{
int readBytes = 0;
while (readBytes < len) {
int ret = in.read(dest, readBytes + offset, len - readBytes);
if (ret == -1) {
finishedReading = true;
return readBytes;
}
readBytes += ret;
}
return readBytes;
}
protected boolean hasNextChunk()
throws IOException
{
if (finishedReading) {
protected boolean hasNextChunk() throws IOException {
if (finishedReading)
return false;
}
uncompressedCursor = 0;
uncompressedLimit = 0;
int readBytes = readNext(header, 0, 4);
if (readBytes < 4) {
return false;
}
int chunkSize = SnappyOutputStream.readInt(header, 0);
if (chunkSize == SnappyCodec.MAGIC_HEADER_HEAD) {
// Concatenated data
int remainingHeaderSize = SnappyCodec.headerSize() - 4;
readBytes = readNext(header, 4, remainingHeaderSize);
if(readBytes < remainingHeaderSize) {
throw new SnappyIOException(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format("Insufficient header size in a concatenated block"));
}
if (isValidHeader(header)) {
return hasNextChunk();
}
else {
int readBytes = 0;
while (readBytes < 4) {
int ret = in.read(chunkSizeBuf, readBytes, 4 - readBytes);
if (ret == -1) {
finishedReading = true;
return false;
}
readBytes += ret;
}
// chunkSize is negative
if (chunkSize < 0) {
throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, "chunkSize is too big or negative : " + chunkSize);
}
// chunkSize is big
if (chunkSize > maxChunkSize) {
throw new SnappyError(SnappyErrorCode.FAILED_TO_UNCOMPRESS, String.format("Received chunkSize %,d is greater than max configured chunk size %,d", chunkSize, maxChunkSize));
}
int chunkSize = SnappyOutputStream.readInt(chunkSizeBuf, 0);
// extend the compressed data buffer size
if (compressed == null || chunkSize > compressed.length) {
// chunkSize exceeds limit
try {
compressed = new byte[chunkSize];
}
catch (java.lang.OutOfMemoryError e) {
throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, e.getMessage());
}
compressed = new byte[chunkSize];
}
readBytes = 0;
while (readBytes < chunkSize) {
int ret = in.read(compressed, readBytes, chunkSize - readBytes);
if (ret == -1) {
if (ret == -1)
break;
}
readBytes += ret;
}
if (readBytes < chunkSize) {
@ -490,19 +379,15 @@ public class SnappyInputStream
* @see java.io.InputStream#read()
*/
@Override
public int read()
throws IOException
{
public int read() throws IOException {
if (uncompressedCursor < uncompressedLimit) {
return uncompressed[uncompressedCursor++] & 0xFF;
}
else {
if (hasNextChunk()) {
if (hasNextChunk())
return read();
}
else {
else
return -1;
}
}
}
@ -510,9 +395,7 @@ public class SnappyInputStream
* @see java.io.InputStream#available()
*/
@Override
public int available()
throws IOException
{
public int available() throws IOException {
if (uncompressedCursor < uncompressedLimit) {
return uncompressedLimit - uncompressedCursor;
}

281
src/main/java/org/xerial/snappy/SnappyLoader.java Normal file → Executable file
View File

@ -19,15 +19,13 @@
// SnappyLoader.java
// Since: 2011/03/29
//
// $URL$
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.*;
import java.net.JarURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.Enumeration;
import java.util.Properties;
import java.util.UUID;
@ -38,10 +36,10 @@ import java.util.UUID;
* user platform (<i>os.name</i> and <i>os.arch</i>). The natively compiled
* libraries bundled to snappy-java contain the codes of the original snappy and
* JNI programs to access Snappy.
* <p/>
*
* In default, no configuration is required to use snappy-java, but you can load
* your own native library created by 'make native' command.
* <p/>
*
* This SnappyLoader searches for native libraries (snappyjava.dll,
* libsnappy.so, etc.) in the following order:
* <ol>
@ -51,85 +49,74 @@ import java.util.UUID;
* <li>(System property: <i>org.xerial.snappy.lib.path</i>)/(System property:
* <i>org.xerial.lib.name</i>)
* <li>One of the libraries embedded in snappy-java-(version).jar extracted into
* (System property: <i>java.io.tmpdir</i>). If
* (System property: <i>java.io.tempdir</i>). If
* <i>org.xerial.snappy.tempdir</i> is set, use this folder instead of
* <i>java.io.tmpdir</i>.
* <i>java.io.tempdir</i>.
* </ol>
* <p/>
*
* <p>
* If you do not want to use folder <i>java.io.tmpdir</i>, set the System
* If you do not want to use folder <i>java.io.tempdir</i>, set the System
* property <i>org.xerial.snappy.tempdir</i>. For example, to use
* <i>/tmp/leo</i> as a temporary folder to copy native libraries, use -D option
* of JVM:
* <p/>
*
* <pre>
* <code>
* java -Dorg.xerial.snappy.tempdir="/tmp/leo" ...
* </code>
* </pre>
* <p/>
*
* </p>
*
*
* @author leo
*
*/
public class SnappyLoader
{
public static final String SNAPPY_SYSTEM_PROPERTIES_FILE = "org-xerial-snappy.properties";
public static final String KEY_SNAPPY_LIB_PATH = "org.xerial.snappy.lib.path";
public static final String KEY_SNAPPY_LIB_NAME = "org.xerial.snappy.lib.name";
public static final String KEY_SNAPPY_PUREJAVA = "org.xerial.snappy.purejava";
public static final String KEY_SNAPPY_TEMPDIR = "org.xerial.snappy.tempdir";
public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib";
public static final String SNAPPY_SYSTEM_PROPERTIES_FILE = "org-xerial-snappy.properties";
public static final String KEY_SNAPPY_LIB_PATH = "org.xerial.snappy.lib.path";
public static final String KEY_SNAPPY_LIB_NAME = "org.xerial.snappy.lib.name";
public static final String KEY_SNAPPY_TEMPDIR = "org.xerial.snappy.tempdir";
public static final String KEY_SNAPPY_USE_SYSTEMLIB = "org.xerial.snappy.use.systemlib";
public static final String KEY_SNAPPY_DISABLE_BUNDLED_LIBS = "org.xerial.snappy.disable.bundled.libs"; // Depreciated, but preserved for backward compatibility
private static boolean isLoaded = false;
private static volatile SnappyApi snappyApi = null;
private static volatile BitShuffleNative bitshuffleApi = null;
private static volatile boolean isLoaded = false;
private static volatile SnappyNative api = null;
private static File nativeLibFile = null;
static void cleanUpExtractedNativeLib()
{
if (nativeLibFile != null && nativeLibFile.exists()) {
boolean deleted = nativeLibFile.delete();
if (!deleted) {
// Deleting native lib has failed, but it's not serious so simply ignore it here
}
snappyApi = null;
bitshuffleApi = null;
}
static void cleanUpExtractedNativeLib() {
if(nativeLibFile != null && nativeLibFile.exists())
nativeLibFile.delete();
}
/**
* Set the `snappyApi` instance.
*
* @param apiImpl
* Set the api instance.
*
* @param nativeCode
*/
static synchronized void setSnappyApi(SnappyApi apiImpl)
static synchronized void setApi(SnappyNative nativeCode)
{
snappyApi = apiImpl;
api = nativeCode;
}
/**
* load system properties when configuration file of the name
* {@link #SNAPPY_SYSTEM_PROPERTIES_FILE} is found
*/
private static void loadSnappySystemProperties()
{
private static void loadSnappySystemProperties() {
try {
InputStream is = Thread.currentThread().getContextClassLoader()
.getResourceAsStream(SNAPPY_SYSTEM_PROPERTIES_FILE);
if (is == null) {
return; // no configuration file is found
}
if (is == null)
return; // no configuration file is found
// Load property file
Properties props = new Properties();
props.load(is);
is.close();
Enumeration<?> names = props.propertyNames();
Enumeration< ? > names = props.propertyNames();
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
if (name.startsWith("org.xerial.snappy.")) {
@ -149,82 +136,69 @@ public class SnappyLoader
loadSnappySystemProperties();
}
static synchronized SnappyApi loadSnappyApi()
static synchronized SnappyNative load()
{
if (snappyApi != null) {
return snappyApi;
}
loadNativeLibrary();
setSnappyApi(new SnappyNative());
return snappyApi;
}
if (api != null)
return api;
static synchronized BitShuffleNative loadBitShuffleApi()
{
if (bitshuffleApi != null) {
return bitshuffleApi;
try {
loadNativeLibrary();
setApi(new SnappyNative());
isLoaded = true;
}
loadNativeLibrary();
bitshuffleApi = new BitShuffleNative();
return bitshuffleApi;
catch (Exception e) {
e.printStackTrace();
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
}
return api;
}
/**
* Load a native library of snappy-java
*/
private synchronized static void loadNativeLibrary()
{
if (!isLoaded) {
try {
nativeLibFile = findNativeLibrary();
if (nativeLibFile != null) {
// Load extracted or specified snappyjava native library.
System.load(nativeLibFile.getAbsolutePath());
} else {
// Load preinstalled snappyjava (in the path -Djava.library.path)
System.loadLibrary("snappyjava");
}
}
catch (Exception e) {
e.printStackTrace();
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, e.getMessage());
}
isLoaded = true;
private static void loadNativeLibrary() {
nativeLibFile = findNativeLibrary();
if (nativeLibFile != null) {
// Load extracted or specified snappyjava native library.
System.load(nativeLibFile.getAbsolutePath());
}
else {
// Load preinstalled snappyjava (in the path -Djava.library.path)
System.loadLibrary("snappyjava");
}
}
private static boolean contentsEquals(InputStream in1, InputStream in2)
throws IOException
{
if (!(in1 instanceof BufferedInputStream)) {
in1 = new BufferedInputStream(in1);
private static boolean contentsEquals(InputStream in1, InputStream in2) throws IOException {
if(!(in1 instanceof BufferedInputStream)) {
in1 = new BufferedInputStream(in1);
}
if (!(in2 instanceof BufferedInputStream)) {
in2 = new BufferedInputStream(in2);
if(!(in2 instanceof BufferedInputStream)) {
in2 = new BufferedInputStream(in2);
}
int ch = in1.read();
while (ch != -1) {
int ch2 = in2.read();
if (ch != ch2) {
while(ch != -1) {
int ch2 = in2.read();
if(ch != ch2)
return false;
}
ch = in1.read();
}
int ch2 = in2.read();
return ch2 == -1;
}
/**
* Extract the specified library file to the target folder
*
*
* @param libFolderForCurrentOS
* @param libraryFileName
* @param targetFolder
* @return
*/
private static File extractLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder)
{
private static File extractLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder) {
String nativeLibraryFilePath = libFolderForCurrentOS + "/" + libraryFileName;
// Attach UUID to the native library file to ensure multiple class loaders can read the libsnappy-java multiple times.
@ -234,61 +208,44 @@ public class SnappyLoader
try {
// Extract a native library file into the target directory
InputStream reader = null;
FileOutputStream writer = null;
InputStream reader = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath);
FileOutputStream writer = new FileOutputStream(extractedLibFile);
try {
reader = getResourceAsInputStream(nativeLibraryFilePath);
try {
writer = new FileOutputStream(extractedLibFile);
byte[] buffer = new byte[8192];
int bytesRead = 0;
while ((bytesRead = reader.read(buffer)) != -1) {
writer.write(buffer, 0, bytesRead);
}
}
finally {
if (writer != null) {
writer.close();
}
byte[] buffer = new byte[8192];
int bytesRead = 0;
while ((bytesRead = reader.read(buffer)) != -1) {
writer.write(buffer, 0, bytesRead);
}
}
finally {
if (reader != null) {
reader.close();
}
// Delete the extracted lib file on JVM exit.
extractedLibFile.deleteOnExit();
if(writer != null)
writer.close();
if(reader != null)
reader.close();
}
// Set executable (x) flag to enable Java to load the native library
boolean success = extractedLibFile.setReadable(true) &&
extractedLibFile.setWritable(true, true) &&
extractedLibFile.setExecutable(true);
if (!success) {
// Setting file flag may fail, but in this case another error will be thrown in later phase
}
extractedLibFile.setReadable(true);
extractedLibFile.setWritable(true, true);
extractedLibFile.setExecutable(true);
// Check whether the contents are properly copied from the resource folder
{
InputStream nativeIn = null;
InputStream extractedLibIn = null;
InputStream nativeIn = SnappyLoader.class.getResourceAsStream(nativeLibraryFilePath);
InputStream extractedLibIn = new FileInputStream(extractedLibFile);
try {
nativeIn = getResourceAsInputStream(nativeLibraryFilePath);
extractedLibIn = new FileInputStream(extractedLibFile);
if (!contentsEquals(nativeIn, extractedLibIn)) {
if(!contentsEquals(nativeIn, extractedLibIn))
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, String.format("Failed to write a native library file at %s", extractedLibFile));
}
}
finally {
if (nativeIn != null) {
if(nativeIn != null)
nativeIn.close();
}
if (extractedLibIn != null) {
if(extractedLibIn != null)
extractedLibIn.close();
}
}
}
@ -300,84 +257,76 @@ public class SnappyLoader
}
}
static File findNativeLibrary()
{
static File findNativeLibrary() {
boolean useSystemLib = Boolean.parseBoolean(System.getProperty(KEY_SNAPPY_USE_SYSTEMLIB, "false"));
boolean disabledBundledLibs = Boolean
.parseBoolean(System.getProperty(KEY_SNAPPY_DISABLE_BUNDLED_LIBS, "false"));
if (useSystemLib || disabledBundledLibs) {
if (useSystemLib || disabledBundledLibs)
return null; // Use a pre-installed libsnappyjava
}
// Try to load the library in org.xerial.snappy.lib.path */
String snappyNativeLibraryPath = System.getProperty(KEY_SNAPPY_LIB_PATH);
String snappyNativeLibraryName = System.getProperty(KEY_SNAPPY_LIB_NAME);
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
if (snappyNativeLibraryName == null) {
// Resolve the library file name with a suffix (e.g., dll, .so, etc.)
if (snappyNativeLibraryName == null)
snappyNativeLibraryName = System.mapLibraryName("snappyjava");
}
if (snappyNativeLibraryPath != null) {
File nativeLib = new File(snappyNativeLibraryPath, snappyNativeLibraryName);
if (nativeLib.exists()) {
if (nativeLib.exists())
return nativeLib;
}
}
// Load an OS-dependent native library inside a jar file
snappyNativeLibraryPath = "/org/xerial/snappy/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
boolean hasNativeLib = hasResource(snappyNativeLibraryPath + "/" + snappyNativeLibraryName);
if (!hasNativeLib) {
if (OSInfo.getOSName().equals("Mac")) {
if(!hasNativeLib) {
if(OSInfo.getOSName().equals("Mac")) {
// Fix for openjdk7 for Mac
String altName = "libsnappyjava.dylib";
if (hasResource(snappyNativeLibraryPath + "/" + altName)) {
String altName = "libsnappyjava.jnilib";
if(hasResource(snappyNativeLibraryPath + "/" + altName)) {
snappyNativeLibraryName = altName;
hasNativeLib = true;
}
}
}
if (!hasNativeLib) {
if(!hasNativeLib) {
String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(), OSInfo.getArchName());
throw new SnappyError(SnappyErrorCode.FAILED_TO_LOAD_NATIVE_LIBRARY, errorMessage);
}
// Temporary folder for the native lib. Use the value of org.xerial.snappy.tempdir or java.io.tmpdir
File tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR, System.getProperty("java.io.tmpdir")));
if (!tempFolder.exists()) {
boolean created = tempFolder.mkdirs();
if (!created) {
// if created == false, it will fail eventually in the later part
}
}
String tempFolder = new File(System.getProperty(KEY_SNAPPY_TEMPDIR,
System.getProperty("java.io.tmpdir"))).getAbsolutePath();
// Extract and load a native library inside the jar file
return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder.getAbsolutePath());
return extractLibraryFile(snappyNativeLibraryPath, snappyNativeLibraryName, tempFolder);
}
private static boolean hasResource(String path)
{
private static boolean hasResource(String path) {
return SnappyLoader.class.getResource(path) != null;
}
/**
* Get the snappy-java version by reading pom.properties embedded in jar.
* This version data is used as a suffix of a dll file extracted from the
* jar.
*
*
* @return the version string
*/
public static String getVersion()
{
public static String getVersion() {
URL versionFile = SnappyLoader.class
.getResource("/META-INF/maven/org.xerial.snappy/snappy-java/pom.properties");
if (versionFile == null) {
if (versionFile == null)
versionFile = SnappyLoader.class.getResource("/org/xerial/snappy/VERSION");
}
String version = "unknown";
try {
@ -385,9 +334,8 @@ public class SnappyLoader
Properties versionData = new Properties();
versionData.load(versionFile.openStream());
version = versionData.getProperty("version", version);
if (version.equals("unknown")) {
version = versionData.getProperty("SNAPPY_VERSION", version);
}
if (version.equals("unknown"))
version = versionData.getProperty("VERSION", version);
version = version.trim().replaceAll("[^0-9M\\.]", "");
}
}
@ -397,15 +345,4 @@ public class SnappyLoader
return version;
}
private static InputStream getResourceAsInputStream(String resourcePath) throws IOException {
URL url = SnappyLoader.class.getResource(resourcePath);
URLConnection connection = url.openConnection();
if (connection instanceof JarURLConnection) {
JarURLConnection jarConnection = (JarURLConnection) connection;
jarConnection.setUseCaches(false); // workaround for https://bugs.openjdk.org/browse/JDK-8205976
return jarConnection.getInputStream();
} else {
return connection.getInputStream();
}
}
}

View File

@ -1,299 +1,309 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
#include <string>
#include <cstring>
#include <snappy.h>
#include "SnappyNative.h"
inline void throw_exception(JNIEnv *env, jobject self, int errorCode)
{
jclass c = env->GetObjectClass(self);
if(c==0)
return;
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
}
JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersion
(JNIEnv * env, jobject self)
{
// TODO: Do we need to read this library version from resources/org/xerial/snappy/VERSION?
return env->NewStringUTF("1.1.3");
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
(JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
size_t compressedLength;
snappy::RawCompress((char*) srcAddr, (size_t) length, (char*) destAddr, &compressedLength);
return (jlong) compressedLength;
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
(JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
size_t uncompressedLength;
snappy::GetUncompressedLength((char*) srcAddr, (size_t) length, &uncompressedLength);
bool ret = snappy::RawUncompress((char*) srcAddr, (size_t) length, (char*) destAddr);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jlong) uncompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: compress
* Signature: (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv* env, jobject self, jobject uncompressed, jint upos, jint ulen, jobject compressed, jint cpos)
{
char* uncompressedBuffer = (char*) env->GetDirectBufferAddress(uncompressed);
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(uncompressedBuffer == 0 || compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t compressedLength;
snappy::RawCompress(uncompressedBuffer + upos, (size_t) ulen, compressedBuffer + cpos, &compressedLength);
return (jint) compressedLength;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_lang_Object_2IILjava_lang_Object_2I
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLen, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
size_t compressedLength;
snappy::RawCompress(in + inputOffset, (size_t) inputLen, out + outputOffset, &compressedLength);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) compressedLength;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_lang_Object_2IILjava_lang_Object_2I
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLength, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
size_t uncompressedLength;
snappy::GetUncompressedLength(in + inputOffset, (size_t) inputLength, &uncompressedLength);
bool ret = snappy::RawUncompress(in + inputOffset, (size_t) inputLength, out + outputOffset);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jint) uncompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: uncompress
* Signature: (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)Z
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen, jobject decompressed, jint dpos)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
char* decompressedBuffer = (char*) env->GetDirectBufferAddress(decompressed);
if(compressedBuffer == 0 || decompressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t decompressedLength;
snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &decompressedLength);
bool ret = snappy::RawUncompress(compressedBuffer + cpos, (size_t) clen, decompressedBuffer + dpos);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jint) decompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: maxCompressedLength
* Signature: (J)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_maxCompressedLength
(JNIEnv *, jobject, jint size)
{
size_t l = snappy::MaxCompressedLength((size_t) size);
return (jint) l;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: getUncompressedLength
* Signature: (Ljava/nio/ByteBuffer;)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t result;
bool ret = snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &result);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_lang_Object_2II
(JNIEnv * env, jobject self, jobject input, jint offset, jint length)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
if(in == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
size_t result;
bool ret = snappy::GetUncompressedLength(in + offset, (size_t) length, &result);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__JJ
(JNIEnv *env, jobject self, jlong inputAddr, jlong len) {
size_t result;
bool ret = snappy::GetUncompressedLength((char*) inputAddr, (size_t) len, &result);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
bool ret = snappy::IsValidCompressedBuffer(compressedBuffer + cpos, (size_t) clen);
return ret;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_lang_Object_2II
(JNIEnv * env, jobject self, jobject input, jint offset, jint length)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
if(in == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
bool ret = snappy::IsValidCompressedBuffer(in + offset, (size_t) length);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
return ret;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__JJJ
(JNIEnv * env, jobject self, jlong inputAddr, jlong offset, jlong length)
{
if(inputAddr == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
bool ret = snappy::IsValidCompressedBuffer((char*) (inputAddr + offset), (size_t) length);
return ret;
}
JNIEXPORT void JNICALL Java_org_xerial_snappy_SnappyNative_arrayCopy
(JNIEnv * env, jobject self, jobject input, jint offset, jint length, jobject output, jint output_offset)
{
char* src = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* dest = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(src == 0 || dest == 0) {
// out of memory
if(src != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, src, 0);
}
if(dest != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
}
throw_exception(env, self, 4);
return;
}
memcpy(dest+output_offset, src+offset, (size_t) length);
env->ReleasePrimitiveArrayCritical((jarray) input, src, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
#include <string>
#include <cstring>
#include <snappy.h>
#include "SnappyNative.h"
void throw_exception(JNIEnv *env, jobject self, int errorCode)
{
jclass c = env->FindClass("org/xerial/snappy/SnappyNative");
if(c==0)
return;
jmethodID mth_throwex = env->GetMethodID(c, "throw_error", "(I)V");
if(mth_throwex == 0)
return;
env->CallVoidMethod(self, mth_throwex, (jint) errorCode);
}
JNIEXPORT jstring JNICALL Java_org_xerial_snappy_SnappyNative_nativeLibraryVersion
(JNIEnv * env, jobject self)
{
return env->NewStringUTF("1.1.0");
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__JJJ
(JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
size_t compressedLength;
snappy::RawCompress((char*) srcAddr, (size_t) length, (char*) destAddr, &compressedLength);
return (jlong) compressedLength;
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__JJJ
(JNIEnv* env, jobject self, jlong srcAddr, jlong length, jlong destAddr) {
size_t uncompressedLength;
snappy::GetUncompressedLength((char*) srcAddr, (size_t) length, &uncompressedLength);
bool ret = snappy::RawUncompress((char*) srcAddr, (size_t) length, (char*) destAddr);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jlong) uncompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: compress
* Signature: (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv* env, jobject self, jobject uncompressed, jint upos, jint ulen, jobject compressed, jint cpos)
{
char* uncompressedBuffer = (char*) env->GetDirectBufferAddress(uncompressed);
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(uncompressedBuffer == 0 || compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t compressedLength;
snappy::RawCompress(uncompressedBuffer + upos, (size_t) ulen, compressedBuffer + cpos, &compressedLength);
return (jint) compressedLength;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawCompress__Ljava_lang_Object_2IILjava_lang_Object_2I
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLen, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
size_t compressedLength;
snappy::RawCompress(in + inputOffset, (size_t) inputLen, out + outputOffset, &compressedLength);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
return (jint) compressedLength;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_lang_Object_2IILjava_lang_Object_2I
(JNIEnv * env, jobject self, jobject input, jint inputOffset, jint inputLength, jobject output, jint outputOffset)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* out = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(in == 0 || out == 0) {
// out of memory
if(in != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
}
if(out != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
}
throw_exception(env, self, 4);
return 0;
}
size_t uncompressedLength;
snappy::GetUncompressedLength(in + inputOffset, (size_t) inputLength, &uncompressedLength);
bool ret = snappy::RawUncompress(in + inputOffset, (size_t) inputLength, out + outputOffset);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, out, 0);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jint) uncompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: uncompress
* Signature: (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)Z
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_rawUncompress__Ljava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2I
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen, jobject decompressed, jint dpos)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
char* decompressedBuffer = (char*) env->GetDirectBufferAddress(decompressed);
if(compressedBuffer == 0 || decompressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t decompressedLength;
snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &decompressedLength);
bool ret = snappy::RawUncompress(compressedBuffer + cpos, (size_t) clen, decompressedBuffer + dpos);
if(!ret) {
throw_exception(env, self, 5);
return 0;
}
return (jint) decompressedLength;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: maxCompressedLength
* Signature: (J)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_maxCompressedLength
(JNIEnv *, jobject, jint size)
{
size_t l = snappy::MaxCompressedLength((size_t) size);
return (jint) l;
}
/*
* Class: org_xerial_snappy_Snappy
* Method: getUncompressedLength
* Signature: (Ljava/nio/ByteBuffer;)J
*/
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
size_t result;
bool ret = snappy::GetUncompressedLength(compressedBuffer + cpos, (size_t) clen, &result);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jint JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__Ljava_lang_Object_2II
(JNIEnv * env, jobject self, jobject input, jint offset, jint length)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
if(in == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
size_t result;
bool ret = snappy::GetUncompressedLength(in + offset, (size_t) length, &result);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jlong JNICALL Java_org_xerial_snappy_SnappyNative_uncompressedLength__JJ
(JNIEnv *env, jobject self, jlong inputAddr, jlong len) {
size_t result;
bool ret = snappy::GetUncompressedLength((char*) inputAddr, (size_t) len, &result);
if(!ret) {
throw_exception(env, self, 2);
return 0;
}
return (jint) result;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_nio_ByteBuffer_2II
(JNIEnv * env, jobject self, jobject compressed, jint cpos, jint clen)
{
char* compressedBuffer = (char*) env->GetDirectBufferAddress(compressed);
if(compressedBuffer == 0) {
throw_exception(env, self, 3);
return (jint) 0;
}
bool ret = snappy::IsValidCompressedBuffer(compressedBuffer + cpos, (size_t) clen);
return ret;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__Ljava_lang_Object_2II
(JNIEnv * env, jobject self, jobject input, jint offset, jint length)
{
char* in = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
if(in == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
bool ret = snappy::IsValidCompressedBuffer(in + offset, (size_t) length);
env->ReleasePrimitiveArrayCritical((jarray) input, in, 0);
return ret;
}
JNIEXPORT jboolean JNICALL Java_org_xerial_snappy_SnappyNative_isValidCompressedBuffer__JJJ
(JNIEnv * env, jobject self, jlong inputAddr, jlong offset, jlong length)
{
if(inputAddr == 0) {
// out of memory
throw_exception(env, self, 4);
return 0;
}
bool ret = snappy::IsValidCompressedBuffer((char*) (inputAddr + offset), (size_t) length);
return ret;
}
JNIEXPORT void JNICALL Java_org_xerial_snappy_SnappyNative_arrayCopy
(JNIEnv * env, jobject self, jobject input, jint offset, jint length, jobject output, jint output_offset)
{
char* src = (char*) env->GetPrimitiveArrayCritical((jarray) input, 0);
char* dest = (char*) env->GetPrimitiveArrayCritical((jarray) output, 0);
if(src == 0 || dest == 0) {
// out of memory
if(src != 0) {
env->ReleasePrimitiveArrayCritical((jarray) input, src, 0);
}
if(dest != 0) {
env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
}
throw_exception(env, self, 4);
return;
}
memcpy(dest+output_offset, src+offset, (size_t) length);
env->ReleasePrimitiveArrayCritical((jarray) input, src, 0);
env->ReleasePrimitiveArrayCritical((jarray) output, dest, 0);
}

0
src/main/java/org/xerial/snappy/SnappyNative.h Normal file → Executable file
View File

View File

@ -1,114 +1,87 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// SnappyNative.java
// Since: 2011/03/30
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* JNI interface of the {@link Snappy} implementation. The native method in this class is
* defined in SnappyNative.h (genereted by javah) and SnappyNative.cpp
* <p/>
* <p>
* <b> DO NOT USE THIS CLASS since the direct use of this class might break the
* native library code loading in {@link SnappyLoader}. </b>
* </p>
*
* @author leo
*/
public class SnappyNative implements SnappyApi
{
public native String nativeLibraryVersion();
// ------------------------------------------------------------------------
// Generic compression/decompression routines.
// ------------------------------------------------------------------------
@Override
public native long rawCompress(long inputAddr, long inputSize, long destAddr)
throws IOException;
@Override
public native long rawUncompress(long inputAddr, long inputSize, long destAddr)
throws IOException;
@Override
public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed,
int outputOffset)
throws IOException;
@Override
public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset)
throws IOException;
@Override
public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed,
int outputOffset)
throws IOException;
@Override
public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException;
// Returns the maximal size of the compressed representation of
// input data that is "source_bytes" bytes in length;
@Override
public native int maxCompressedLength(int source_bytes);
// This operation takes O(1) time.
@Override
public native int uncompressedLength(ByteBuffer compressed, int offset, int len)
throws IOException;
@Override
public native int uncompressedLength(Object input, int offset, int len)
throws IOException;
@Override
public native long uncompressedLength(long inputAddr, long len)
throws IOException;
@Override
public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len)
throws IOException;
@Override
public native boolean isValidCompressedBuffer(Object input, int offset, int len)
throws IOException;
@Override
public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len)
throws IOException;
@Override
public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset)
throws IOException;
public void throw_error(int errorCode)
throws IOException
{
throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode));
}
}
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
//--------------------------------------
// snappy-java Project
//
// SnappyNative.java
// Since: 2011/03/30
//
// $URL$
// $Author$
//--------------------------------------
package org.xerial.snappy;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* JNI interface of the {@link Snappy} implementation. The native method in this class is
* defined in SnappyNative.h (genereted by javah) and SnappyNative.cpp
*
* <p>
* <b> DO NOT USE THIS CLASS since the direct use of this class might break the
* native library code loading in {@link SnappyLoader}. </b>
* </p>
*
* @author leo
*
*/
public class SnappyNative
{
public native String nativeLibraryVersion();
// ------------------------------------------------------------------------
// Generic compression/decompression routines.
// ------------------------------------------------------------------------
public native long rawCompress(long inputAddr, long inputSize, long destAddr) throws IOException;
public native long rawUncompress(long inputAddr, long inputSize, long destAddr) throws IOException;
public native int rawCompress(ByteBuffer input, int inputOffset, int inputLength, ByteBuffer compressed,
int outputOffset) throws IOException;
public native int rawCompress(Object input, int inputOffset, int inputByteLength, Object output, int outputOffset) throws IOException;
public native int rawUncompress(ByteBuffer compressed, int inputOffset, int inputLength, ByteBuffer uncompressed,
int outputOffset) throws IOException;
public native int rawUncompress(Object input, int inputOffset, int inputLength, Object output, int outputOffset)
throws IOException;
// Returns the maximal size of the compressed representation of
// input data that is "source_bytes" bytes in length;
public native int maxCompressedLength(int source_bytes);
// This operation takes O(1) time.
public native int uncompressedLength(ByteBuffer compressed, int offset, int len) throws IOException;
public native int uncompressedLength(Object input, int offset, int len) throws IOException;
public native long uncompressedLength(long inputAddr, long len) throws IOException;
public native boolean isValidCompressedBuffer(ByteBuffer compressed, int offset, int len) throws IOException;
public native boolean isValidCompressedBuffer(Object input, int offset, int len) throws IOException;
public native boolean isValidCompressedBuffer(long inputAddr, long offset, long len) throws IOException;
public native void arrayCopy(Object src, int offset, int byteLength, Object dest, int dOffset) throws IOException;
public void throw_error(int errorCode) throws IOException {
throw new IOException(String.format("%s(%d)", SnappyErrorCode.getErrorMessage(errorCode), errorCode));
}
}

View File

@ -41,7 +41,6 @@ import java.io.OutputStream;
* The output data format is:
* <ol>
* <li>snappy codec header defined in {@link SnappyCodec} (8 bytes)
* <li>codec version (4 bytes integer, at least 1) and compatible version (4 bytes integer)
* <li>compressed block 1 : a pair of (compressed data size [4 byte integer.
* Big-endian], compressed data...)
* <li>compressed block 2
@ -56,12 +55,9 @@ import java.io.OutputStream;
*
* @author leo
*/
public class SnappyOutputStream
extends OutputStream
{
public static final int MAX_BLOCK_SIZE = 512 * 1024 * 1024; // 512 MiB
public static final int MIN_BLOCK_SIZE = 1 * 1024;
public static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
public class SnappyOutputStream extends OutputStream {
static final int MIN_BLOCK_SIZE = 1 * 1024;
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
protected final OutputStream out;
private final int blockSize;
@ -69,36 +65,27 @@ public class SnappyOutputStream
private final BufferAllocator inputBufferAllocator;
private final BufferAllocator outputBufferAllocator;
// The input and output buffer fields are set to null when closing this stream:
protected byte[] inputBuffer;
protected byte[] outputBuffer;
protected final byte[] inputBuffer;
protected final byte[] outputBuffer;
private int inputCursor = 0;
private int outputCursor = 0;
private boolean headerWritten;
private boolean closed;
public SnappyOutputStream(OutputStream out)
{
public SnappyOutputStream(OutputStream out) {
this(out, DEFAULT_BLOCK_SIZE);
}
/**
* @param out
* @param blockSize byte size of the internal buffer size
* @throws IllegalArgumentException when blockSize is larger than 512 MiB
* @throws IOException
*/
public SnappyOutputStream(OutputStream out, int blockSize)
{
this(out, blockSize, CachedBufferAllocator.getBufferAllocatorFactory());
public SnappyOutputStream(OutputStream out, int blockSize) {
this(out, blockSize, CachedBufferAllocator.factory);
}
public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactory bufferAllocatorFactory)
{
public SnappyOutputStream(OutputStream out, int blockSize, BufferAllocatorFactory bufferAllocatorFactory) {
this.out = out;
this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
if (this.blockSize > MAX_BLOCK_SIZE){
throw new IllegalArgumentException(String.format("Provided chunk size %,d larger than max %,d", this.blockSize, MAX_BLOCK_SIZE));
}
int inputSize = blockSize;
int outputSize = SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(blockSize);
@ -106,103 +93,77 @@ public class SnappyOutputStream
this.outputBufferAllocator = bufferAllocatorFactory.getBufferAllocator(outputSize);
inputBuffer = inputBufferAllocator.allocate(inputSize);
outputBuffer = outputBufferAllocator.allocate(outputSize);
outputBuffer = inputBufferAllocator.allocate(outputSize);
outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
}
/* (non-Javadoc)
* @see java.io.OutputStream#write(byte[], int, int)
*/
@Override
public void write(byte[] b, int byteOffset, int byteLength)
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
int cursor = 0;
while (cursor < byteLength) {
int readLen = Math.min(byteLength - cursor, blockSize - inputCursor);
// copy the input data to uncompressed buffer
if (readLen > 0) {
System.arraycopy(b, byteOffset + cursor, inputBuffer, inputCursor, readLen);
inputCursor += readLen;
}
if (inputCursor < blockSize) {
return;
}
compressInput();
cursor += readLen;
}
public void write(byte[] b, int off, int len) throws IOException {
rawWrite(b, off, len);
}
/**
* Compress the input long array data
*
* @param d input array
* @param d input array
* @param off offset in the array
* @param len the number of elements in the array to copy
* @throws IOException
*/
public void write(long[] d, int off, int len)
throws IOException
{
public void write(long[] d, int off, int len) throws IOException {
rawWrite(d, off * 8, len * 8);
}
/**
* Compress the input double array data
*
* @param f input array
* @param f input array
* @param off offset in the array
* @param len the number of elements in the array to copy
* @throws IOException
*/
public void write(double[] f, int off, int len)
throws IOException
{
public void write(double[] f, int off, int len) throws IOException {
rawWrite(f, off * 8, len * 8);
}
/**
* Compress the input float array data
*
* @param f input array
* @param f input array
* @param off offset in the array
* @param len the number of elements in the array to copy
* @throws IOException
*/
public void write(float[] f, int off, int len)
throws IOException
{
public void write(float[] f, int off, int len) throws IOException {
rawWrite(f, off * 4, len * 4);
}
/**
* Compress the input int array data
*
* @param f input array
* @param f input array
* @param off offset in the array
* @param len the number of elements in the array to copy
* @throws IOException
*/
public void write(int[] f, int off, int len)
throws IOException
{
public void write(int[] f, int off, int len) throws IOException {
rawWrite(f, off * 4, len * 4);
}
/**
* Compress the input short array data
*
* @param f input array
* @param f input array
* @param off offset in the array
* @param len the number of elements in the array to copy
* @throws IOException
*/
public void write(short[] f, int off, int len)
throws IOException
{
public void write(short[] f, int off, int len) throws IOException {
rawWrite(f, off * 2, len * 2);
}
@ -212,9 +173,7 @@ public class SnappyOutputStream
* @param d
* @throws IOException
*/
public void write(long[] d)
throws IOException
{
public void write(long[] d) throws IOException {
write(d, 0, d.length);
}
@ -224,9 +183,7 @@ public class SnappyOutputStream
* @param f
* @throws IOException
*/
public void write(double[] f)
throws IOException
{
public void write(double[] f) throws IOException {
write(f, 0, f.length);
}
@ -236,9 +193,7 @@ public class SnappyOutputStream
* @param f
* @throws IOException
*/
public void write(float[] f)
throws IOException
{
public void write(float[] f) throws IOException {
write(f, 0, f.length);
}
@ -248,9 +203,7 @@ public class SnappyOutputStream
* @param f
* @throws IOException
*/
public void write(int[] f)
throws IOException
{
public void write(int[] f) throws IOException {
write(f, 0, f.length);
}
@ -260,14 +213,11 @@ public class SnappyOutputStream
* @param f
* @throws IOException
*/
public void write(short[] f)
throws IOException
{
public void write(short[] f) throws IOException {
write(f, 0, f.length);
}
private boolean hasSufficientOutputBufferFor(int inputSize)
{
private boolean hasSufficientOutputBufferFor(int inputSize) {
int maxCompressedSize = Snappy.maxCompressedLength(inputSize);
return maxCompressedSize < outputBuffer.length - outputCursor - 4;
}
@ -275,31 +225,31 @@ public class SnappyOutputStream
/**
* Compress the raw byte array data.
*
* @param array array data of any type (e.g., byte[], float[], long[], ...)
* @param array array data of any type (e.g., byte[], float[], long[], ...)
* @param byteOffset
* @param byteLength
* @throws IOException
*/
public void rawWrite(Object array, int byteOffset, int byteLength)
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
int cursor = 0;
while (cursor < byteLength) {
int readLen = Math.min(byteLength - cursor, blockSize - inputCursor);
// copy the input data to uncompressed buffer
if (readLen > 0) {
Snappy.arrayCopy(array, byteOffset + cursor, readLen, inputBuffer, inputCursor);
inputCursor += readLen;
}
if (inputCursor < blockSize) {
return;
}
public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException {
compressInput();
cursor += readLen;
if(inputCursor + byteLength < MIN_BLOCK_SIZE) {
// copy the input data to uncompressed buffer
Snappy.arrayCopy(array, byteOffset, byteLength, inputBuffer, inputCursor);
inputCursor += byteLength;
return;
}
compressInput();
for(int readBytes = 0; readBytes < byteLength; ) {
int inputLen = Math.min(blockSize, byteLength - readBytes);
if(!hasSufficientOutputBufferFor(inputLen)) {
dumpOutput();
}
int compressedSize = Snappy.rawCompress(array, byteOffset + readBytes, inputLen, outputBuffer, outputCursor + 4);
writeInt(outputBuffer, outputCursor, compressedSize);
outputCursor += 4 + compressedSize;
readBytes += inputLen;
}
}
@ -313,13 +263,8 @@ public class SnappyOutputStream
* @see java.io.OutputStream#write(int)
*/
@Override
public void write(int b)
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
if (inputCursor >= inputBuffer.length) {
public void write(int b) throws IOException {
if(inputCursor >= inputBuffer.length) {
compressInput();
}
inputBuffer[inputCursor++] = (byte) b;
@ -329,27 +274,20 @@ public class SnappyOutputStream
* @see java.io.OutputStream#flush()
*/
@Override
public void flush()
throws IOException
{
if (closed) {
throw new IOException("Stream is closed");
}
public void flush() throws IOException {
compressInput();
dumpOutput();
out.flush();
}
static void writeInt(byte[] dst, int offset, int v)
{
static void writeInt(byte[] dst, int offset, int v) {
dst[offset] = (byte) ((v >> 24) & 0xFF);
dst[offset + 1] = (byte) ((v >> 16) & 0xFF);
dst[offset + 2] = (byte) ((v >> 8) & 0xFF);
dst[offset + 3] = (byte) ((v >> 0) & 0xFF);
}
static int readInt(byte[] buffer, int pos)
{
static int readInt(byte[] buffer, int pos) {
int b1 = (buffer[pos] & 0xFF) << 24;
int b2 = (buffer[pos + 1] & 0xFF) << 16;
int b3 = (buffer[pos + 2] & 0xFF) << 8;
@ -357,30 +295,22 @@ public class SnappyOutputStream
return b1 | b2 | b3 | b4;
}
protected void dumpOutput()
throws IOException
{
if (outputCursor > 0) {
protected void dumpOutput() throws IOException {
if(outputCursor > 0) {
out.write(outputBuffer, 0, outputCursor);
outputCursor = 0;
}
}
protected void compressInput()
throws IOException
{
if (!headerWritten) {
outputCursor = writeHeader();
headerWritten = true;
}
if (inputCursor <= 0) {
protected void compressInput() throws IOException {
if(inputCursor <= 0) {
return; // no need to dump
}
// Compress and dump the buffer content
if (!hasSufficientOutputBufferFor(inputCursor)) {
if(!hasSufficientOutputBufferFor(inputCursor)) {
dumpOutput();
}
writeBlockPreemble();
int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4);
// Write compressed data size
writeInt(outputBuffer, outputCursor, compressedSize);
@ -388,24 +318,6 @@ public class SnappyOutputStream
inputCursor = 0;
}
protected int writeHeader(){
return SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
}
/**
* Optionally write a preemble before a block. Hadoop requires the actual block data size being written. This base
* implementation does nothing. Derive implementation can call {@code writeCurrentDataSize()}.
*/
protected void writeBlockPreemble()
{
// do nothing
}
protected void writeCurrentDataSize(){
writeInt(outputBuffer, outputCursor, inputCursor);
outputCursor += 4;
}
/**
* close the stream
*/
@ -413,22 +325,14 @@ public class SnappyOutputStream
* @see java.io.OutputStream#close()
*/
@Override
public void close()
throws IOException
{
if (closed) {
return;
}
public void close() throws IOException {
try {
flush();
out.close();
}
finally {
closed = true;
} finally {
inputBufferAllocator.release(inputBuffer);
outputBufferAllocator.release(outputBuffer);
inputBuffer = null;
outputBuffer = null;
}
}
}

View File

@ -0,0 +1,2 @@
VERSION=1.1.1

View File

@ -3,10 +3,9 @@ package org.xerial.snappy.buffer;
/**
* BufferAllocator interface. The implementation of this interface must be thread-safe
*/
public interface BufferAllocator
{
public interface BufferAllocator {
public byte[] allocate(int size);
public void release(byte[] buffer);
}

View File

@ -3,8 +3,7 @@ package org.xerial.snappy.buffer;
/**
*
*/
public interface BufferAllocatorFactory
{
public interface BufferAllocatorFactory {
BufferAllocator getBufferAllocator(int minSize);
}

View File

@ -6,45 +6,29 @@ import java.util.*;
/**
* Cached buffer
*/
public class CachedBufferAllocator
implements BufferAllocator
{
private static BufferAllocatorFactory factory = new BufferAllocatorFactory()
{
public class CachedBufferAllocator implements BufferAllocator {
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() {
@Override
public BufferAllocator getBufferAllocator(int bufferSize)
{
public BufferAllocator getBufferAllocator(int bufferSize) {
return CachedBufferAllocator.getAllocator(bufferSize);
}
};
public static void setBufferAllocatorFactory(BufferAllocatorFactory factory)
{
assert (factory != null);
CachedBufferAllocator.factory = factory;
}
public static BufferAllocatorFactory getBufferAllocatorFactory()
{
return factory;
}
/**
* Use SoftReference so that having this queueTable does not prevent the GC of CachedBufferAllocator instances
*/
private static final Map<Integer, SoftReference<CachedBufferAllocator>> queueTable = new HashMap<Integer, SoftReference<CachedBufferAllocator>>();
public static Map<Integer, SoftReference<CachedBufferAllocator>> queueTable = new HashMap<Integer, SoftReference<CachedBufferAllocator>>();
private final int bufferSize;
private final Deque<byte[]> bufferQueue;
public CachedBufferAllocator(int bufferSize)
{
public CachedBufferAllocator(int bufferSize) {
this.bufferSize = bufferSize;
this.bufferQueue = new ArrayDeque<byte[]>();
}
public static synchronized CachedBufferAllocator getAllocator(int bufferSize)
{
public static synchronized CachedBufferAllocator getAllocator(int bufferSize) {
CachedBufferAllocator result = null;
if (queueTable.containsKey(bufferSize)) {
@ -58,10 +42,9 @@ public class CachedBufferAllocator
}
@Override
public byte[] allocate(int size)
{
synchronized (this) {
if (bufferQueue.isEmpty()) {
public byte[] allocate(int size) {
synchronized(this) {
if(bufferQueue.isEmpty()) {
return new byte[size];
}
else {
@ -69,11 +52,9 @@ public class CachedBufferAllocator
}
}
}
@Override
public void release(byte[] buffer)
{
synchronized (this) {
public void release(byte[] buffer) {
synchronized(this) {
bufferQueue.addLast(buffer);
}
}

View File

@ -3,30 +3,24 @@ package org.xerial.snappy.buffer;
/**
* Simple buffer allocator, which does not reuse the allocated buffer
*/
public class DefaultBufferAllocator
implements BufferAllocator
{
public class DefaultBufferAllocator implements BufferAllocator {
public static BufferAllocatorFactory factory = new BufferAllocatorFactory()
{
public static BufferAllocatorFactory factory = new BufferAllocatorFactory() {
public BufferAllocator singleton = new DefaultBufferAllocator();
@Override
public BufferAllocator getBufferAllocator(int bufferSize)
{
public BufferAllocator getBufferAllocator(int bufferSize) {
return singleton;
}
};
@Override
public byte[] allocate(int size)
{
public byte[] allocate(int size) {
return new byte[size];
}
@Override
public void release(byte[] buffer)
{
public void release(byte[] buffer) {
// do nothing
}
}

View File

@ -1,44 +1,44 @@
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
/**
* Snappy API for compressing/decompressing data.
* <p/>
* Usage
* First, import {@link org.xerial.snappy.Snappy} in your Java code:
* <code>
* <pre>
* import org.xerial.snappy.Snappy;
* </pre>
* </code>
* Then use {@link org.xerial.snappy.Snappy#compress(byte[])} and {@link org.xerial.snappy.Snappy#uncompress(byte[])}:
* <code>
* <pre>
* String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of Snappy, a fast compresser/decompresser.";
* byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
* byte[] uncompressed = Snappy.uncompress(compressed);
* String result = new String(uncompressed, "UTF-8");
* System.out.println(result);
* </pre>
* </code>
* <p/>
* <p>In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used. </p>
* <p/>
* <h3>Stream-based API</h3>
* Stream-based compressor/decompressor SnappyOutputStream, SnappyInputStream are also available for reading/writing large data sets.
*/
package org.xerial.snappy;
/*--------------------------------------------------------------------------
* Copyright 2011 Taro L. Saito
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*--------------------------------------------------------------------------*/
/**
* Snappy API for compressing/decompressing data.
*
* Usage
* First, import {@link org.xerial.snappy.Snappy} in your Java code:
* <code>
* <pre>
* import org.xerial.snappy.Snappy;
* </pre>
* </code>
* Then use {@link org.xerial.snappy.Snappy#compress(byte[])} and {@link org.xerial.snappy.Snappy#uncompress(byte[])}:
* <code>
* <pre>
* String input = "Hello snappy-java! Snappy-java is a JNI-based wrapper of Snappy, a fast compresser/decompresser.";
* byte[] compressed = Snappy.compress(input.getBytes("UTF-8"));
* byte[] uncompressed = Snappy.uncompress(compressed);
* String result = new String(uncompressed, "UTF-8");
* System.out.println(result);
* </pre>
* </code>
*
* <p>In addition, high-level methods (Snappy.compress(String), Snappy.compress(float[] ..) etc. ) and low-level ones (e.g. Snappy.rawCompress(.. ), Snappy.rawUncompress(..), etc.), which minimize memory copies, can be used. </p>
*
* <h3>Stream-based API</h3>
* Stream-based compressor/decompressor SnappyOutputStream, SnappyInputStream are also available for reading/writing large data sets.
*/
package org.xerial.snappy;

View File

@ -1,53 +0,0 @@
package org.xerial.snappy.pool;
import java.nio.ByteBuffer;
/**
* Makes various types of buffers available for use and potential re-use.
*
* <p>
* Implementations must be safe for concurrent use by multiple threads.
* </p>
*
* @author Brett Okken
*/
public interface BufferPool {
/**
* Returns a {@code byte[]} of <i>size</i> or greater length.
* @param size The minimum size array required. Must be {@code >= 0}.
* @return A {@code byte[]} with length of at least <i>size</i>.
* @see #releaseArray(byte[])
*/
public byte[] allocateArray(int size);
/**
* Returns instance to pool for potential future reuse.
* <p>
* Must not be returned more than 1 time. Must not be used by caller after return.
* </p>
* @param buffer Instance to return to pool. Must not be {@code null}.
* Must not be returned more than 1 time. Must not be used by caller after return.
*/
public void releaseArray(byte[] buffer);
/**
* Returns a {@link ByteBuffer#allocateDirect(int) direct ByteBuffer} of <i>size</i> or
* greater {@link ByteBuffer#capacity() capacity}.
* @param size The minimum size buffer required. Must be {@code >= 0}.
* @return A {@code ByteBuffer} of <i>size</i> or greater {@link ByteBuffer#capacity() capacity}.
* @see #releaseDirect(ByteBuffer)
* @see ByteBuffer#allocateDirect(int)
*/
public ByteBuffer allocateDirect(int size);
/**
* Returns instance to pool for potential future reuse.
* <p>
* Must not be returned more than 1 time. Must not be used by caller after return.
* </p>
* @param buffer Instance to return to pool. Must not be {@code null}.
* Must not be returned more than 1 time. Must not be used by caller after return.
*/
public void releaseDirect(ByteBuffer buffer);
}

View File

@ -1,216 +0,0 @@
package org.xerial.snappy.pool;
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentMap;
/**
* A {@link BufferPool} implementation which caches values at fixed sizes.
* <p>
* Pooled instances are held as {@link SoftReference} to allow GC if necessary.
* </p>
* <p>
* The current fixed sizes are calculated as follows:
* <ul>
* <li>Values < 4KB return 4KB</li>
* <li>4KB - 32KB to 2KB</li>
* <li>32KB - 512KB to 16KB</li>
* <li>512KB - 2MB to 128KB</li>
* <li>2MB - 16MB to 512KB</li>
* <li>16MB - 128MB to 4MB</li>
* <li>128MB - 512MB to 16MB</li>
* <li>512MB - 1.5 GB to 128MB</li>
* <li>Values > 1.5GB return {@link Integer#MAX_VALUE}</li>
* </ul>
* </p>
* @author Brett Okken
*/
public final class CachingBufferPool implements BufferPool {
private static interface IntFunction<E> {
public E create(int size);
}
private static final IntFunction<byte[]> ARRAY_FUNCTION = new IntFunction<byte[]>() {
@Override
public byte[] create(int size) {
return new byte[size];
}
};
private static final IntFunction<ByteBuffer> DBB_FUNCTION = new IntFunction<ByteBuffer>() {
@Override
public ByteBuffer create(int size) {
return ByteBuffer.allocateDirect(size);
}
};
private static final CachingBufferPool INSTANCE = new CachingBufferPool();
private final ConcurrentMap<Integer, ConcurrentLinkedDeque<SoftReference<byte[]>>> bytes = new ConcurrentHashMap<>();
private final ConcurrentMap<Integer, ConcurrentLinkedDeque<SoftReference<ByteBuffer>>> buffers = new ConcurrentHashMap<>();
private CachingBufferPool() {
}
/**
* Returns instance of {@link CachingBufferPool} for using cached buffers.
* @return instance of {@link CachingBufferPool} for using cached buffers.
*/
public static BufferPool getInstance() {
return INSTANCE;
}
/**
* {@inheritDoc}
*/
@Override
public byte[] allocateArray(int size) {
if (size <= 0) {
throw new IllegalArgumentException("size is invalid: " + size);
}
return getOrCreate(size, bytes, ARRAY_FUNCTION);
}
/**
* {@inheritDoc}
*/
@Override
public void releaseArray(byte[] buffer) {
if (buffer == null) {
throw new IllegalArgumentException("buffer is null");
}
returnValue(buffer, buffer.length, bytes);
}
/**
* {@inheritDoc}
*/
@Override
public ByteBuffer allocateDirect(int size) {
if (size <= 0) {
throw new IllegalArgumentException("size is invalid: " + size);
}
return getOrCreate(size, buffers, DBB_FUNCTION);
}
/**
* {@inheritDoc}
*/
@Override
public void releaseDirect(ByteBuffer buffer) {
if (buffer == null) {
throw new IllegalArgumentException("buffer is null");
}
buffer.clear();
returnValue(buffer, buffer.capacity(), buffers);
}
private static <E> E getOrCreate(final int size, final ConcurrentMap<Integer, ConcurrentLinkedDeque<SoftReference<E>>> map, final IntFunction<E> creator) {
assert size > 0;
final int adjustedSize = adjustSize(size);
final ConcurrentLinkedDeque<SoftReference<E>> queue = optimisticGetEntry(adjustedSize, map);
SoftReference<E> entry;
while ((entry = queue.pollFirst()) != null) {
final E val = entry.get();
if (val != null) {
return val;
}
}
return creator.create(adjustedSize);
}
/*
* This is package scope to allow direct unit testing.
*/
static int adjustSize(int size) {
assert size > 0;
switch (Integer.numberOfLeadingZeros(size)) {
case 1: // 1GB - 2GB
case 2: // 512MB
//if 512MB - 1.5 GB round to nearest 128 MB (2^27), else Integer.MAX_VALUE
return size <= 0x6000_0000 ? roundToPowers(size, 27) : Integer.MAX_VALUE;
case 3: //256MB
case 4: //128MB
//if 128MB - 512MB, round to nearest 16 MB
return roundToPowers(size, 24);
case 5: // 64MB
case 6: // 32MB
case 7: // 16MB
//if 16MB - 128MB, round to nearest 4MB
return roundToPowers(size, 22);
case 8: // 8MB
case 9: // 4MB
case 10: // 2MB
//if 2MB - 16MB, round to nearest 512KB
return roundToPowers(size, 19);
case 11: // 1MB
case 12: //512KB
//if 512KB - 2MB, round to nearest 128KB
return roundToPowers(size, 17);
case 13: //256KB
case 14: //128KB
case 15: // 64KB
case 16: // 32KB
//if 32KB to 512KB, round to nearest 16KB
return roundToPowers(size, 14);
case 17: // 16KB
case 18: // 8KB
case 19: // 4KB
// if 4KB - 32KB, round to nearest 2KB
return roundToPowers(size, 11);
default:
return 4 * 1024;
}
}
private static int roundToPowers(int number, int bits) {
final int mask = (0x7FFF_FFFF >> bits) << bits;
final int floor = number & mask;
return floor == number ? number : floor + (1 << bits);
}
private static <E> ConcurrentLinkedDeque<SoftReference<E>> optimisticGetEntry(Integer key, ConcurrentMap<Integer, ConcurrentLinkedDeque<SoftReference<E>>> map) {
ConcurrentLinkedDeque<SoftReference<E>> val = map.get(key);
if (val == null) {
map.putIfAbsent(key, new ConcurrentLinkedDeque<SoftReference<E>>());
val = map.get(key);
}
return val;
}
private static <E> void returnValue(E value, Integer size, ConcurrentMap<Integer, ConcurrentLinkedDeque<SoftReference<E>>> map) {
final ConcurrentLinkedDeque<SoftReference<E>> queue = map.get(size);
//no queue will exist if buffer was not originally obtained from this class
if (queue != null) {
//push this value onto deque first so that concurrent request can use it
queue.addFirst(new SoftReference<E>(value));
//purge oldest entries have lost references
SoftReference<E> entry;
boolean lastEmpty = true;
while(lastEmpty && (entry = queue.peekLast()) != null) {
if (entry.get() == null) {
queue.removeLastOccurrence(entry);
} else {
lastEmpty = false;
}
}
}
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return "CachingBufferPool [bytes=" + this.bytes + ", buffers=" + this.buffers + "]";
}
}

View File

@ -1,37 +0,0 @@
package org.xerial.snappy.pool;
/**
* Manages implementation of {@link BufferPool} to use by default. Setting the system property {@link #DISABLE_CACHING_PROPERTY} to {@code true} will
* cause the {@link QuiescentBufferPool} to be used by default. Otherwise, {@link CachingBufferPool} will be used by default.
* {@link #setDefaultPool(BufferPool)} can be used to explicitly control the implementation to use.
*/
public final class DefaultPoolFactory {
/**
* Name of system property to disable use of {@link CachingBufferPool} by default.
*/
public static final String DISABLE_CACHING_PROPERTY = "org.xerial.snappy.pool.disable";
private static volatile BufferPool defaultPool = "true".equalsIgnoreCase(System.getProperty(DISABLE_CACHING_PROPERTY))
? QuiescentBufferPool.getInstance()
: CachingBufferPool.getInstance();
/**
* @return The default instance to use.
*/
public static BufferPool getDefaultPool() {
return defaultPool;
}
/**
* Sets the default instance to use.
* @param pool The default instance to use. Must not be {@code null}.
* @see #getDefaultPool()
*/
public static void setDefaultPool(BufferPool pool) {
if (pool == null) {
throw new IllegalArgumentException("pool is null");
}
defaultPool = pool;
}
}

View File

@ -1,150 +0,0 @@
package org.xerial.snappy.pool;
import static java.lang.invoke.MethodHandles.constant;
import static java.lang.invoke.MethodHandles.dropArguments;
import static java.lang.invoke.MethodHandles.filterReturnValue;
import static java.lang.invoke.MethodHandles.guardWithTest;
import static java.lang.invoke.MethodHandles.lookup;
import static java.lang.invoke.MethodType.methodType;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles.Lookup;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Utility to facilitate disposing of direct byte buffer instances.
*/
final class DirectByteBuffers {
/**
* Sun specific mechanisms to clean up resources associated with direct byte buffers.
*/
@SuppressWarnings("unchecked")
static final Class<? extends ByteBuffer> DIRECT_BUFFER_CLAZZ = (Class<? extends ByteBuffer>) lookupClassQuietly("java.nio.DirectByteBuffer");
static final MethodHandle CLEAN_HANDLE;
static {
// this approach is based off that used by apache lucene and documented here: https://issues.apache.org/jira/browse/LUCENE-6989
// and https://github.com/apache/lucene-solr/blob/7e03427fa14a024ce257babcb8362d2451941e21/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
MethodHandle cleanHandle = null;
try {
final PrivilegedExceptionAction<MethodHandle> action = new PrivilegedExceptionAction<MethodHandle>() {
@Override
public MethodHandle run() throws Exception {
MethodHandle handle = null;
if (DIRECT_BUFFER_CLAZZ != null) {
final Lookup lookup = lookup();
try {
// sun.misc.Unsafe unmapping (Java 9+)
final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
// first check if Unsafe has the right method, otherwise we can give up
// without doing any security critical stuff:
final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", methodType(void.class, ByteBuffer.class));
// fetch the unsafe instance and bind it to the virtual MH:
final Field f = unsafeClass.getDeclaredField("theUnsafe");
f.setAccessible(true);
final Object theUnsafe = f.get(null);
handle = unmapper.bindTo(theUnsafe);
} catch (Exception e) {
Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "unable to use java 9 Unsafe.invokeCleaner", e);
// sun.misc.Cleaner unmapping (Java 8 and older)
final Method m = DIRECT_BUFFER_CLAZZ.getMethod("cleaner");
m.setAccessible(true);
final MethodHandle directBufferCleanerMethod = lookup.unreflect(m);
final Class<?> cleanerClass = directBufferCleanerMethod.type().returnType();
/*
* "Compile" a MethodHandle that basically is equivalent to the following code:
* void unmapper(ByteBuffer byteBuffer)
* {
* sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner();
* if (nonNull(cleaner))
* {
* cleaner.clean();
* }
* else
* {
* // the noop is needed because MethodHandles#guardWithTest always needs ELSE
* noop(cleaner);
* }
* }
*/
final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class));
final MethodHandle nonNullTest = lookup.findStatic(DirectByteBuffers.class, "nonNull", methodType(boolean.class, Object.class)).asType(methodType(boolean.class, cleanerClass));
final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass);
handle = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)).asType(methodType(void.class, ByteBuffer.class));
}
}
return handle;
}
};
cleanHandle = AccessController.doPrivileged(action);
} catch (Throwable t) {
Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Exception occurred attempting to lookup Sun specific DirectByteBuffer cleaner classes.", t);
}
CLEAN_HANDLE = cleanHandle;
}
private static Class<?> lookupClassQuietly(String name)
{
try {
return DirectByteBuffers.class.getClassLoader().loadClass(name);
}
catch (Throwable t) {
Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Did not find requested class: " + name, t);
}
return null;
}
static boolean nonNull(Object o) {
return o != null;
}
/**
* Provides jvm implementation specific operation to aggressively release resources associated with <i>buffer</i>.
*
* @param buffer The {@code ByteBuffer} to release. Must not be {@code null}. Must be {@link ByteBuffer#isDirect() direct}.
*/
public static void releaseDirectByteBuffer(final ByteBuffer buffer)
{
assert buffer != null && buffer.isDirect();
if (CLEAN_HANDLE != null && DIRECT_BUFFER_CLAZZ.isInstance(buffer)) {
try {
final PrivilegedExceptionAction<Void> pea = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try {
CLEAN_HANDLE.invokeExact(buffer);
} catch (Exception e) {
throw e;
} catch (Throwable t) {
//this will be an error
throw new RuntimeException(t);
}
return null;
}
};
AccessController.doPrivileged(pea);
} catch (Throwable t) {
Logger.getLogger(DirectByteBuffers.class.getName()).log(Level.FINE, "Exception occurred attempting to clean up Sun specific DirectByteBuffer.", t);
}
}
}
}

View File

@ -1,55 +0,0 @@
package org.xerial.snappy.pool;
import java.nio.ByteBuffer;
/**
* A {@link BufferPool} implementation which does no pooling. New instances will be created for each call to allocate.
* @author Brett Okken
*/
public final class QuiescentBufferPool implements BufferPool {
private static final QuiescentBufferPool INSTANCE = new QuiescentBufferPool();
private QuiescentBufferPool() {
}
/**
* @return Instance of {@link BufferPool} which does no caching/reuse of instances.
*/
public static BufferPool getInstance() {
return INSTANCE;
}
/**
* Creates a new {@code byte[]} of <i>size</i>.
*/
@Override
public byte[] allocateArray(int size) {
return new byte[size];
}
/**
* Does nothing.
*/
@Override
public void releaseArray(byte[] buffer) {
}
/**
* {@link ByteBuffer#allocateDirect(int) Allocates} a direct {@link ByteBuffer} of <i>size</i>.
*/
@Override
public ByteBuffer allocateDirect(int size) {
return ByteBuffer.allocateDirect(size);
}
/**
* Aggressively releases native resources associated with <i>buffer</i>.
*/
@Override
public void releaseDirect(ByteBuffer buffer) {
assert buffer != null && buffer.isDirect();
DirectByteBuffers.releaseDirectByteBuffer(buffer);
}
}

View File

@ -1,11 +0,0 @@
{
"resources": {
"includes": [
{"pattern":".*libsnappyjava.dylib"},
{"pattern":".*libsnappyjava.so"},
{"pattern":".*snappyjava.dll"},
{"pattern":".*libsnappyjava.a"}
]
},
"bundles": []
}

View File

@ -1,2 +0,0 @@
SNAPPY_VERSION=1.1.10
BITSHUFFLE_VERSION=0.3.4

Some files were not shown because too many files have changed in this diff Show More