mirror of
https://github.com/xerial/snappy-java.git
synced 2025-07-25 06:54:51 +02:00
Sanitize code
This commit is contained in:
Taro L. Saito
parent
7adf0c1dbc
commit
50164bc210
@ -2,9 +2,10 @@ The snappy-java is a Java port of the snappy
|
||||
<http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google.
|
||||
|
||||
## Features
|
||||
* Fast compression/decompression tailored to 64-bit CPU architecture.
|
||||
* Fast compression/decompression tailored to 64-bit CPU architecture.
|
||||
* JNI-based implementation to achieve comparable performance to the native C++ version.
|
||||
* Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.).
|
||||
* Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.).
|
||||
* Supporting compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
|
||||
* Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
|
||||
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
|
||||
* [Framing-format support](http://snappy.googlecode.com/svn/trunk/framing_format.txt) (Since 1.1.0 version)
|
||||
|
||||
@ -1,314 +1,313 @@
|
||||
/*--------------------------------------------------------------------------
|
||||
* Copyright 2011 Taro L. Saito
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*--------------------------------------------------------------------------*/
|
||||
//--------------------------------------
|
||||
// XerialJ
|
||||
//
|
||||
// SnappyOutputStream.java
|
||||
// Since: 2011/03/31 17:44:10
|
||||
//
|
||||
// $URL$
|
||||
// $Author$
|
||||
//--------------------------------------
|
||||
package org.xerial.snappy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
/**
|
||||
* This class implements a stream filter for writing compressed data using
|
||||
* Snappy.
|
||||
* <p>
|
||||
* The input data is blocked into 32kb size (in default), and each block is
|
||||
* compressed and then passed to the given {@link OutputStream}.
|
||||
* </p>
|
||||
* The output data format is:
|
||||
* <ol>
|
||||
* <li>snappy codec header defined in {@link SnappyCodec} (8 bytes)
|
||||
* <li>compressed block 1 : a pair of (compressed data size [4 byte integer.
|
||||
* Big-endian], compressed data...)
|
||||
* <li>compressed block 2
|
||||
* <li>...
|
||||
* </ol>
|
||||
* <p/>
|
||||
* Note that the compressed data created by {@link SnappyOutputStream} cannot be
|
||||
* uncompressed by {@link Snappy#uncompress(byte[])} since the output formats of
|
||||
* {@link Snappy#compress(byte[])} and {@link SnappyOutputStream} are different.
|
||||
* Use {@link SnappyInputStream} for uncompress the data generated by
|
||||
* {@link SnappyOutputStream}.
|
||||
*
|
||||
* @author leo
|
||||
*/
|
||||
public class SnappyOutputStream extends OutputStream {
|
||||
static final int MIN_BLOCK_SIZE = 1 * 1024;
|
||||
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
|
||||
|
||||
protected final OutputStream out;
|
||||
private final int blockSize;
|
||||
private int inputCursor = 0;
|
||||
protected byte[] uncompressed;
|
||||
private int outputCursor = 0;
|
||||
protected byte[] outputBuffer;
|
||||
|
||||
public SnappyOutputStream(OutputStream out) {
|
||||
this(out, DEFAULT_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param out
|
||||
* @param blockSize byte size of the internal buffer size
|
||||
* @throws IOException
|
||||
*/
|
||||
public SnappyOutputStream(OutputStream out, int blockSize) {
|
||||
this.out = out;
|
||||
this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
|
||||
uncompressed = new byte[blockSize];
|
||||
outputBuffer = new byte[SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(blockSize)];
|
||||
outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
rawWrite(b, off, len);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input long array data
|
||||
*
|
||||
* @param d input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(long[] d, int off, int len) throws IOException {
|
||||
rawWrite(d, off * 8, len * 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input double array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(double[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 8, len * 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input float array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(float[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 4, len * 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input int array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(int[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 4, len * 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input short array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(short[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 2, len * 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param d
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(long[] d) throws IOException {
|
||||
write(d, 0, d.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(double[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(float[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(int[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(short[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
private boolean hasSufficientOutputBufferFor(int inputSize) {
|
||||
int maxCompressedSize = Snappy.maxCompressedLength(inputSize);
|
||||
return maxCompressedSize < outputBuffer.length - outputCursor - 4;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the raw byte array data.
|
||||
*
|
||||
* @param array array data of any type (e.g., byte[], float[], long[], ...)
|
||||
* @param byteOffset
|
||||
* @param byteLength
|
||||
* @throws IOException
|
||||
*/
|
||||
public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException {
|
||||
|
||||
if(inputCursor + byteLength < MIN_BLOCK_SIZE) {
|
||||
// copy the input data to uncompressed buffer
|
||||
Snappy.arrayCopy(array, byteOffset, byteLength, uncompressed, inputCursor);
|
||||
inputCursor += byteLength;
|
||||
return;
|
||||
}
|
||||
|
||||
compressInput();
|
||||
|
||||
for(int readBytes = 0; readBytes < byteLength; ) {
|
||||
int inputLen = Math.min(blockSize, byteLength - readBytes);
|
||||
if(!hasSufficientOutputBufferFor(inputLen)) {
|
||||
dumpOutput();
|
||||
}
|
||||
int compressedSize = Snappy.rawCompress(array, byteOffset + readBytes, inputLen, outputBuffer, outputCursor + 4);
|
||||
writeInt(outputBuffer, outputCursor, compressedSize);
|
||||
outputCursor += 4 + compressedSize;
|
||||
readBytes += inputLen;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the specified byte to this output stream. The general contract for
|
||||
* write is that one byte is written to the output stream. The byte to be
|
||||
* written is the eight low-order bits of the argument b. The 24 high-order
|
||||
* bits of b are ignored.
|
||||
*/
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
if(inputCursor >= uncompressed.length) {
|
||||
compressInput();
|
||||
}
|
||||
uncompressed[inputCursor++] = (byte) b;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#flush()
|
||||
*/
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
compressInput();
|
||||
dumpOutput();
|
||||
out.flush();
|
||||
}
|
||||
|
||||
static void writeInt(byte[] dst, int offset, int v) {
|
||||
int p = offset;
|
||||
dst[offset++] = (byte) ((v >> 24) & 0xFF);
|
||||
dst[offset++] = (byte) ((v >> 16) & 0xFF);
|
||||
dst[offset++] = (byte) ((v >> 8) & 0xFF);
|
||||
dst[offset++] = (byte) ((v >> 0) & 0xFF);
|
||||
}
|
||||
|
||||
static int readInt(byte[] buffer, int pos) {
|
||||
int b1 = (buffer[pos] & 0xFF) << 24;
|
||||
int b2 = (buffer[pos + 1] & 0xFF) << 16;
|
||||
int b3 = (buffer[pos + 2] & 0xFF) << 8;
|
||||
int b4 = buffer[pos + 3] & 0xFF;
|
||||
return b1 | b2 | b3 | b4;
|
||||
}
|
||||
|
||||
protected void dumpOutput() throws IOException {
|
||||
if(outputCursor > 0) {
|
||||
out.write(outputBuffer, 0, outputCursor);
|
||||
outputCursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
protected void compressInput() throws IOException {
|
||||
if(inputCursor <= 0) {
|
||||
return; // no need to dump
|
||||
}
|
||||
|
||||
// Compress and dump the buffer content
|
||||
if(!hasSufficientOutputBufferFor(inputCursor)) {
|
||||
dumpOutput();
|
||||
}
|
||||
int compressedSize = Snappy.compress(uncompressed, 0, inputCursor, outputBuffer, outputCursor + 4);
|
||||
// Write compressed data size
|
||||
writeInt(outputBuffer, outputCursor, compressedSize);
|
||||
outputCursor += 4 + compressedSize;
|
||||
inputCursor = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* close the stream
|
||||
*/
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
flush();
|
||||
out.close();
|
||||
}
|
||||
|
||||
}
|
||||
/*--------------------------------------------------------------------------
|
||||
* Copyright 2011 Taro L. Saito
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*--------------------------------------------------------------------------*/
|
||||
//--------------------------------------
|
||||
// XerialJ
|
||||
//
|
||||
// SnappyOutputStream.java
|
||||
// Since: 2011/03/31 17:44:10
|
||||
//
|
||||
// $URL$
|
||||
// $Author$
|
||||
//--------------------------------------
|
||||
package org.xerial.snappy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
/**
|
||||
* This class implements a stream filter for writing compressed data using
|
||||
* Snappy.
|
||||
* <p>
|
||||
* The input data is blocked into 32kb size (in default), and each block is
|
||||
* compressed and then passed to the given {@link OutputStream}.
|
||||
* </p>
|
||||
* The output data format is:
|
||||
* <ol>
|
||||
* <li>snappy codec header defined in {@link SnappyCodec} (8 bytes)
|
||||
* <li>compressed block 1 : a pair of (compressed data size [4 byte integer.
|
||||
* Big-endian], compressed data...)
|
||||
* <li>compressed block 2
|
||||
* <li>...
|
||||
* </ol>
|
||||
* <p/>
|
||||
* Note that the compressed data created by {@link SnappyOutputStream} cannot be
|
||||
* uncompressed by {@link Snappy#uncompress(byte[])} since the output formats of
|
||||
* {@link Snappy#compress(byte[])} and {@link SnappyOutputStream} are different.
|
||||
* Use {@link SnappyInputStream} for uncompress the data generated by
|
||||
* {@link SnappyOutputStream}.
|
||||
*
|
||||
* @author leo
|
||||
*/
|
||||
public class SnappyOutputStream extends OutputStream {
|
||||
static final int MIN_BLOCK_SIZE = 1 * 1024;
|
||||
static final int DEFAULT_BLOCK_SIZE = 32 * 1024; // Use 32kb for the default block size
|
||||
|
||||
protected final OutputStream out;
|
||||
private final int blockSize;
|
||||
private int inputCursor = 0;
|
||||
protected byte[] uncompressed;
|
||||
private int outputCursor = 0;
|
||||
protected byte[] outputBuffer;
|
||||
|
||||
public SnappyOutputStream(OutputStream out) {
|
||||
this(out, DEFAULT_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param out
|
||||
* @param blockSize byte size of the internal buffer size
|
||||
* @throws IOException
|
||||
*/
|
||||
public SnappyOutputStream(OutputStream out, int blockSize) {
|
||||
this.out = out;
|
||||
this.blockSize = Math.max(MIN_BLOCK_SIZE, blockSize);
|
||||
uncompressed = new byte[this.blockSize];
|
||||
outputBuffer = new byte[SnappyCodec.HEADER_SIZE + 4 + Snappy.maxCompressedLength(this.blockSize)];
|
||||
outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
rawWrite(b, off, len);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input long array data
|
||||
*
|
||||
* @param d input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(long[] d, int off, int len) throws IOException {
|
||||
rawWrite(d, off * 8, len * 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input double array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(double[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 8, len * 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input float array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(float[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 4, len * 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input int array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(int[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 4, len * 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input short array data
|
||||
*
|
||||
* @param f input array
|
||||
* @param off offset in the array
|
||||
* @param len the number of elements in the array to copy
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(short[] f, int off, int len) throws IOException {
|
||||
rawWrite(f, off * 2, len * 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param d
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(long[] d) throws IOException {
|
||||
write(d, 0, d.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(double[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(float[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(int[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the input array data
|
||||
*
|
||||
* @param f
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(short[] f) throws IOException {
|
||||
write(f, 0, f.length);
|
||||
}
|
||||
|
||||
private boolean hasSufficientOutputBufferFor(int inputSize) {
|
||||
int maxCompressedSize = Snappy.maxCompressedLength(inputSize);
|
||||
return maxCompressedSize < outputBuffer.length - outputCursor - 4;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress the raw byte array data.
|
||||
*
|
||||
* @param array array data of any type (e.g., byte[], float[], long[], ...)
|
||||
* @param byteOffset
|
||||
* @param byteLength
|
||||
* @throws IOException
|
||||
*/
|
||||
public void rawWrite(Object array, int byteOffset, int byteLength) throws IOException {
|
||||
|
||||
if(inputCursor + byteLength < MIN_BLOCK_SIZE) {
|
||||
// copy the input data to uncompressed buffer
|
||||
Snappy.arrayCopy(array, byteOffset, byteLength, uncompressed, inputCursor);
|
||||
inputCursor += byteLength;
|
||||
return;
|
||||
}
|
||||
|
||||
compressInput();
|
||||
|
||||
for(int readBytes = 0; readBytes < byteLength; ) {
|
||||
int inputLen = Math.min(blockSize, byteLength - readBytes);
|
||||
if(!hasSufficientOutputBufferFor(inputLen)) {
|
||||
dumpOutput();
|
||||
}
|
||||
int compressedSize = Snappy.rawCompress(array, byteOffset + readBytes, inputLen, outputBuffer, outputCursor + 4);
|
||||
writeInt(outputBuffer, outputCursor, compressedSize);
|
||||
outputCursor += 4 + compressedSize;
|
||||
readBytes += inputLen;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the specified byte to this output stream. The general contract for
|
||||
* write is that one byte is written to the output stream. The byte to be
|
||||
* written is the eight low-order bits of the argument b. The 24 high-order
|
||||
* bits of b are ignored.
|
||||
*/
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
if(inputCursor >= uncompressed.length) {
|
||||
compressInput();
|
||||
}
|
||||
uncompressed[inputCursor++] = (byte) b;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#flush()
|
||||
*/
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
compressInput();
|
||||
dumpOutput();
|
||||
out.flush();
|
||||
}
|
||||
|
||||
static void writeInt(byte[] dst, int offset, int v) {
|
||||
dst[offset] = (byte) ((v >> 24) & 0xFF);
|
||||
dst[offset+1] = (byte) ((v >> 16) & 0xFF);
|
||||
dst[offset+2] = (byte) ((v >> 8) & 0xFF);
|
||||
dst[offset+3] = (byte) ((v >> 0) & 0xFF);
|
||||
}
|
||||
|
||||
static int readInt(byte[] buffer, int pos) {
|
||||
int b1 = (buffer[pos] & 0xFF) << 24;
|
||||
int b2 = (buffer[pos + 1] & 0xFF) << 16;
|
||||
int b3 = (buffer[pos + 2] & 0xFF) << 8;
|
||||
int b4 = buffer[pos + 3] & 0xFF;
|
||||
return b1 | b2 | b3 | b4;
|
||||
}
|
||||
|
||||
protected void dumpOutput() throws IOException {
|
||||
if(outputCursor > 0) {
|
||||
out.write(outputBuffer, 0, outputCursor);
|
||||
outputCursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
protected void compressInput() throws IOException {
|
||||
if(inputCursor <= 0) {
|
||||
return; // no need to dump
|
||||
}
|
||||
|
||||
// Compress and dump the buffer content
|
||||
if(!hasSufficientOutputBufferFor(inputCursor)) {
|
||||
dumpOutput();
|
||||
}
|
||||
int compressedSize = Snappy.compress(uncompressed, 0, inputCursor, outputBuffer, outputCursor + 4);
|
||||
// Write compressed data size
|
||||
writeInt(outputBuffer, outputCursor, compressedSize);
|
||||
outputCursor += 4 + compressedSize;
|
||||
inputCursor = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* close the stream
|
||||
*/
|
||||
/* (non-Javadoc)
|
||||
* @see java.io.OutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
flush();
|
||||
out.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,188 +1,211 @@
|
||||
/*--------------------------------------------------------------------------
|
||||
* Copyright 2011 Taro L. Saito
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*--------------------------------------------------------------------------*/
|
||||
//--------------------------------------
|
||||
// XerialJ
|
||||
//
|
||||
// SnappyOutputStreamTest.java
|
||||
// Since: 2011/03/31 18:26:31
|
||||
//
|
||||
// $URL$
|
||||
// $Author$
|
||||
//--------------------------------------
|
||||
package org.xerial.snappy;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.xerial.util.FileResource;
|
||||
import org.xerial.util.log.Logger;
|
||||
|
||||
public class SnappyOutputStreamTest
|
||||
{
|
||||
private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class);
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
||||
SnappyOutputStream sout = new SnappyOutputStream(buf);
|
||||
|
||||
BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class,
|
||||
"alice29.txt").openStream());
|
||||
assertNotNull(input);
|
||||
|
||||
ByteArrayOutputStream orig = new ByteArrayOutputStream();
|
||||
byte[] tmp = new byte[1024];
|
||||
for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) {
|
||||
sout.write(tmp, 0, readBytes);
|
||||
orig.write(tmp, 0, readBytes); // preserve the original data
|
||||
}
|
||||
input.close();
|
||||
sout.flush();
|
||||
orig.flush();
|
||||
|
||||
int compressedSize = buf.size();
|
||||
_logger.debug("compressed size: " + compressedSize);
|
||||
|
||||
ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
|
||||
byte[] compressed = buf.toByteArray();
|
||||
// decompress
|
||||
for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length;) {
|
||||
int chunkSize = SnappyOutputStream.readInt(compressed, cursor);
|
||||
cursor += 4;
|
||||
byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)];
|
||||
int decompressedSize = Snappy.uncompress(compressed, cursor, chunkSize, tmpOut, 0);
|
||||
cursor += chunkSize;
|
||||
|
||||
decompressed.write(tmpOut);
|
||||
}
|
||||
decompressed.flush();
|
||||
assertEquals(orig.size(), decompressed.size());
|
||||
assertArrayEquals(orig.toByteArray(), decompressed.toByteArray());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bufferSize() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b, 500);
|
||||
final int bytesToWrite = 5000;
|
||||
byte[] orig = new byte[bytesToWrite];
|
||||
for (int i = 0; i < 5000; ++i) {
|
||||
byte v = (byte) (i % 128);
|
||||
orig[i] = v;
|
||||
os.write(v);
|
||||
}
|
||||
os.close();
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
byte[] buf = new byte[bytesToWrite / 101];
|
||||
while (is.read(buf) != -1) {}
|
||||
is.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void longArrayCompress() throws Exception {
|
||||
long[] l = new long[10];
|
||||
for (int i = 0; i < l.length; ++i) {
|
||||
l[i] = i % 3 + i * 11;
|
||||
}
|
||||
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
os.write(l);
|
||||
os.close();
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
long[] l2 = new long[10];
|
||||
int readBytes = is.read(l2);
|
||||
is.close();
|
||||
|
||||
assertEquals(10 * 8, readBytes);
|
||||
assertArrayEquals(l, l2);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeDoubleArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
double[] orig = new double[] { 1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
double[] uncompressed = new double[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed, 0.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeFloatArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
float[] orig = new float[] { 1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
float[] uncompressed = new float[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed, 0.0f);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeIntArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
int[] orig = new int[] { 0, -1, -34, 43, 234, 34324, -234 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
int[] uncompressed = new int[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeShortArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
short[] orig = new short[] { 0, -1, -34, 43, 234, 324, -234 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
short[] uncompressed = new short[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed);
|
||||
}
|
||||
|
||||
}
|
||||
/*--------------------------------------------------------------------------
|
||||
* Copyright 2011 Taro L. Saito
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*--------------------------------------------------------------------------*/
|
||||
//--------------------------------------
|
||||
// XerialJ
|
||||
//
|
||||
// SnappyOutputStreamTest.java
|
||||
// Since: 2011/03/31 18:26:31
|
||||
//
|
||||
// $URL$
|
||||
// $Author$
|
||||
//--------------------------------------
|
||||
package org.xerial.snappy;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.xerial.util.FileResource;
|
||||
import org.xerial.util.log.Logger;
|
||||
|
||||
public class SnappyOutputStreamTest
|
||||
{
|
||||
private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class);
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
||||
SnappyOutputStream sout = new SnappyOutputStream(buf);
|
||||
|
||||
BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class,
|
||||
"alice29.txt").openStream());
|
||||
assertNotNull(input);
|
||||
|
||||
ByteArrayOutputStream orig = new ByteArrayOutputStream();
|
||||
byte[] tmp = new byte[1024];
|
||||
for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) {
|
||||
sout.write(tmp, 0, readBytes);
|
||||
orig.write(tmp, 0, readBytes); // preserve the original data
|
||||
}
|
||||
input.close();
|
||||
sout.flush();
|
||||
orig.flush();
|
||||
|
||||
int compressedSize = buf.size();
|
||||
_logger.debug("compressed size: " + compressedSize);
|
||||
|
||||
ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
|
||||
byte[] compressed = buf.toByteArray();
|
||||
// decompress
|
||||
for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length;) {
|
||||
int chunkSize = SnappyOutputStream.readInt(compressed, cursor);
|
||||
cursor += 4;
|
||||
byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)];
|
||||
int decompressedSize = Snappy.uncompress(compressed, cursor, chunkSize, tmpOut, 0);
|
||||
cursor += chunkSize;
|
||||
|
||||
decompressed.write(tmpOut);
|
||||
}
|
||||
decompressed.flush();
|
||||
assertEquals(orig.size(), decompressed.size());
|
||||
assertArrayEquals(orig.toByteArray(), decompressed.toByteArray());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bufferSize() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b, 1500);
|
||||
final int bytesToWrite = 5000;
|
||||
byte[] orig = new byte[bytesToWrite];
|
||||
for (int i = 0; i < 5000; ++i) {
|
||||
byte v = (byte) (i % 128);
|
||||
orig[i] = v;
|
||||
os.write(v);
|
||||
}
|
||||
os.close();
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
byte[] buf = new byte[bytesToWrite / 101];
|
||||
while (is.read(buf) != -1) {}
|
||||
is.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void smallWrites() throws Exception {
|
||||
|
||||
byte[] orig = CalgaryTest.readFile("alice29.txt");
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream out = new SnappyOutputStream(b);
|
||||
|
||||
for(byte c : orig) {
|
||||
out.write(c);
|
||||
}
|
||||
out.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
byte[] decompressed = new byte[orig.length];
|
||||
int cursor = 0;
|
||||
int readLen = 0;
|
||||
for(int i=0; i < decompressed.length && (readLen = is.read(decompressed, i, decompressed.length-i)) != -1; ) {
|
||||
i += readLen;
|
||||
}
|
||||
is.close();
|
||||
assertArrayEquals(orig, decompressed);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void longArrayCompress() throws Exception {
|
||||
long[] l = new long[10];
|
||||
for (int i = 0; i < l.length; ++i) {
|
||||
l[i] = i % 3 + i * 11;
|
||||
}
|
||||
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
os.write(l);
|
||||
os.close();
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
long[] l2 = new long[10];
|
||||
int readBytes = is.read(l2);
|
||||
is.close();
|
||||
|
||||
assertEquals(10 * 8, readBytes);
|
||||
assertArrayEquals(l, l2);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeDoubleArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
double[] orig = new double[] { 1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
double[] uncompressed = new double[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed, 0.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeFloatArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
float[] orig = new float[] { 1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
float[] uncompressed = new float[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed, 0.0f);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeIntArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
int[] orig = new int[] { 0, -1, -34, 43, 234, 34324, -234 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
int[] uncompressed = new int[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeShortArray() throws Exception {
|
||||
ByteArrayOutputStream b = new ByteArrayOutputStream();
|
||||
SnappyOutputStream os = new SnappyOutputStream(b);
|
||||
|
||||
short[] orig = new short[] { 0, -1, -34, 43, 234, 324, -234 };
|
||||
os.write(orig);
|
||||
os.close();
|
||||
|
||||
SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
|
||||
short[] uncompressed = new short[orig.length];
|
||||
is.read(uncompressed);
|
||||
is.close();
|
||||
|
||||
assertArrayEquals(orig, uncompressed);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user