Prefer use of java.util.zip.CRC32C for Framed format (#269)
Co-authored-by: BO8979 <BO8979@W1971362.northamerica.cerner.net>
This commit is contained in:
parent
110727ed69
commit
822513dbe6
|
@ -48,8 +48,7 @@ public class PureJavaCrc32C
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
public long getValue()
|
public long getValue()
|
||||||
{
|
{
|
||||||
long ret = crc;
|
return (~crc) & 0xffffffffL;
|
||||||
return (~ret) & 0xffffffffL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
|
|
|
@ -4,8 +4,15 @@
|
||||||
package org.xerial.snappy;
|
package org.xerial.snappy;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.lang.invoke.MethodHandle;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
|
import java.lang.invoke.MethodType;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.nio.channels.ReadableByteChannel;
|
import java.nio.channels.ReadableByteChannel;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
import java.util.logging.Level;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
import java.util.zip.Checksum;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constants and utilities for implementing x-snappy-framed.
|
* Constants and utilities for implementing x-snappy-framed.
|
||||||
|
@ -23,6 +30,39 @@ final class SnappyFramed
|
||||||
|
|
||||||
private static final int MASK_DELTA = 0xa282ead8;
|
private static final int MASK_DELTA = 0xa282ead8;
|
||||||
|
|
||||||
|
private static final Supplier<Checksum> CHECKSUM_SUPPLIER;
|
||||||
|
|
||||||
|
static
|
||||||
|
{
|
||||||
|
Supplier<Checksum> supplier = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
final Class crc32cClazz = Class.forName("java.util.zip.CRC32C");
|
||||||
|
final MethodHandles.Lookup lookup = MethodHandles.publicLookup();
|
||||||
|
|
||||||
|
final MethodHandle conHandle = lookup.findConstructor(crc32cClazz, MethodType.methodType(void.class))
|
||||||
|
.asType(MethodType.methodType(Checksum.class));
|
||||||
|
supplier = () -> {
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return (Checksum) conHandle.invokeExact();
|
||||||
|
}
|
||||||
|
catch (Throwable e)
|
||||||
|
{
|
||||||
|
throw new IllegalStateException(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
catch(Throwable t)
|
||||||
|
{
|
||||||
|
Logger.getLogger(SnappyFramed.class.getName())
|
||||||
|
.log(Level.FINE, "java.util.zip.CRC32C not loaded, using PureJavaCrc32C", t);
|
||||||
|
supplier = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
CHECKSUM_SUPPLIER = supplier != null ? supplier : PureJavaCrc32C::new;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The header consists of the stream identifier flag, 3 bytes indicating a
|
* The header consists of the stream identifier flag, 3 bytes indicating a
|
||||||
* length of 6, and "sNaPpY" in ASCII.
|
* length of 6, and "sNaPpY" in ASCII.
|
||||||
|
@ -31,16 +71,16 @@ final class SnappyFramed
|
||||||
(byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
|
(byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
|
||||||
0x50, 0x70, 0x59};
|
0x50, 0x70, 0x59};
|
||||||
|
|
||||||
public static int maskedCrc32c(byte[] data)
|
public static Checksum getCRC32C()
|
||||||
{
|
{
|
||||||
return maskedCrc32c(data, 0, data.length);
|
return CHECKSUM_SUPPLIER.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static int maskedCrc32c(byte[] data, int offset, int length)
|
public static int maskedCrc32c(Checksum crc32c, byte[] data, int offset, int length)
|
||||||
{
|
{
|
||||||
final PureJavaCrc32C crc32c = new PureJavaCrc32C();
|
crc32c.reset();
|
||||||
crc32c.update(data, offset, length);
|
crc32c.update(data, offset, length);
|
||||||
return mask(crc32c.getIntegerValue());
|
return mask((int) crc32c.getValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.nio.channels.ClosedChannelException;
|
||||||
import java.nio.channels.ReadableByteChannel;
|
import java.nio.channels.ReadableByteChannel;
|
||||||
import java.nio.channels.WritableByteChannel;
|
import java.nio.channels.WritableByteChannel;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.zip.Checksum;
|
||||||
|
|
||||||
import org.xerial.snappy.pool.BufferPool;
|
import org.xerial.snappy.pool.BufferPool;
|
||||||
import org.xerial.snappy.pool.DefaultPoolFactory;
|
import org.xerial.snappy.pool.DefaultPoolFactory;
|
||||||
|
@ -40,6 +41,7 @@ public final class SnappyFramedInputStream
|
||||||
ReadableByteChannel
|
ReadableByteChannel
|
||||||
{
|
{
|
||||||
|
|
||||||
|
private final Checksum crc32 = SnappyFramed.getCRC32C();
|
||||||
private final ReadableByteChannel rbc;
|
private final ReadableByteChannel rbc;
|
||||||
private final ByteBuffer frameHeader;
|
private final ByteBuffer frameHeader;
|
||||||
private final boolean verifyChecksums;
|
private final boolean verifyChecksums;
|
||||||
|
@ -572,7 +574,7 @@ public final class SnappyFramedInputStream
|
||||||
}
|
}
|
||||||
|
|
||||||
if (verifyChecksums) {
|
if (verifyChecksums) {
|
||||||
final int actualCrc32c = SnappyFramed.maskedCrc32c(buffer,
|
final int actualCrc32c = SnappyFramed.maskedCrc32c(crc32, buffer,
|
||||||
position, valid - position);
|
position, valid - position);
|
||||||
if (frameData.checkSum != actualCrc32c) {
|
if (frameData.checkSum != actualCrc32c) {
|
||||||
throw new IOException("Corrupt input: invalid checksum");
|
throw new IOException("Corrupt input: invalid checksum");
|
||||||
|
|
|
@ -17,6 +17,7 @@ import java.nio.channels.Channels;
|
||||||
import java.nio.channels.ClosedChannelException;
|
import java.nio.channels.ClosedChannelException;
|
||||||
import java.nio.channels.ReadableByteChannel;
|
import java.nio.channels.ReadableByteChannel;
|
||||||
import java.nio.channels.WritableByteChannel;
|
import java.nio.channels.WritableByteChannel;
|
||||||
|
import java.util.zip.Checksum;
|
||||||
|
|
||||||
import org.xerial.snappy.pool.BufferPool;
|
import org.xerial.snappy.pool.BufferPool;
|
||||||
import org.xerial.snappy.pool.DefaultPoolFactory;
|
import org.xerial.snappy.pool.DefaultPoolFactory;
|
||||||
|
@ -59,6 +60,7 @@ public final class SnappyFramedOutputStream
|
||||||
*/
|
*/
|
||||||
public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d;
|
public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d;
|
||||||
|
|
||||||
|
private final Checksum crc32 = SnappyFramed.getCRC32C();
|
||||||
private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order(
|
private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order(
|
||||||
ByteOrder.LITTLE_ENDIAN);
|
ByteOrder.LITTLE_ENDIAN);
|
||||||
private final BufferPool bufferPool;
|
private final BufferPool bufferPool;
|
||||||
|
@ -503,7 +505,7 @@ public final class SnappyFramedOutputStream
|
||||||
final int length = buffer.remaining();
|
final int length = buffer.remaining();
|
||||||
|
|
||||||
// crc is based on the user supplied input data
|
// crc is based on the user supplied input data
|
||||||
final int crc32c = maskedCrc32c(input, 0, length);
|
final int crc32c = maskedCrc32c(crc32, input, 0, length);
|
||||||
|
|
||||||
directInputBuffer.clear();
|
directInputBuffer.clear();
|
||||||
directInputBuffer.put(buffer);
|
directInputBuffer.put(buffer);
|
||||||
|
|
|
@ -9,7 +9,6 @@ import static org.junit.Assert.fail;
|
||||||
import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
|
import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
|
||||||
import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
|
import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
|
||||||
import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
|
import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
|
||||||
import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
|
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
|
@ -418,4 +417,9 @@ public class SnappyFramedStreamTest
|
||||||
assertEquals(random.length, length);
|
assertEquals(random.length, length);
|
||||||
return random;
|
return random;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static int maskedCrc32c(byte[] data)
|
||||||
|
{
|
||||||
|
return SnappyFramed.maskedCrc32c(new PureJavaCrc32C(), data, 0, data.length);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue