Prefer use of java.util.zip.CRC32C for Framed format (#269)

Co-authored-by: BO8979 <BO8979@W1971362.northamerica.cerner.net>
This commit is contained in:
Brett Okken 2021-01-20 13:30:19 -06:00 committed by GitHub
parent 110727ed69
commit 822513dbe6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 1426 additions and 1379 deletions

View File

@ -48,8 +48,7 @@ public class PureJavaCrc32C
/** {@inheritDoc} */ /** {@inheritDoc} */
public long getValue() public long getValue()
{ {
long ret = crc; return (~crc) & 0xffffffffL;
return (~ret) & 0xffffffffL;
} }
/** {@inheritDoc} */ /** {@inheritDoc} */

View File

@ -4,8 +4,15 @@
package org.xerial.snappy; package org.xerial.snappy;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel; import java.nio.channels.ReadableByteChannel;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.Checksum;
/** /**
* Constants and utilities for implementing x-snappy-framed. * Constants and utilities for implementing x-snappy-framed.
@ -23,6 +30,39 @@ final class SnappyFramed
private static final int MASK_DELTA = 0xa282ead8; private static final int MASK_DELTA = 0xa282ead8;
private static final Supplier<Checksum> CHECKSUM_SUPPLIER;
static
{
Supplier<Checksum> supplier = null;
try
{
final Class crc32cClazz = Class.forName("java.util.zip.CRC32C");
final MethodHandles.Lookup lookup = MethodHandles.publicLookup();
final MethodHandle conHandle = lookup.findConstructor(crc32cClazz, MethodType.methodType(void.class))
.asType(MethodType.methodType(Checksum.class));
supplier = () -> {
try
{
return (Checksum) conHandle.invokeExact();
}
catch (Throwable e)
{
throw new IllegalStateException(e);
}
};
}
catch(Throwable t)
{
Logger.getLogger(SnappyFramed.class.getName())
.log(Level.FINE, "java.util.zip.CRC32C not loaded, using PureJavaCrc32C", t);
supplier = null;
}
CHECKSUM_SUPPLIER = supplier != null ? supplier : PureJavaCrc32C::new;
}
/** /**
* The header consists of the stream identifier flag, 3 bytes indicating a * The header consists of the stream identifier flag, 3 bytes indicating a
* length of 6, and "sNaPpY" in ASCII. * length of 6, and "sNaPpY" in ASCII.
@ -31,16 +71,16 @@ final class SnappyFramed
(byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
0x50, 0x70, 0x59}; 0x50, 0x70, 0x59};
public static int maskedCrc32c(byte[] data) public static Checksum getCRC32C()
{ {
return maskedCrc32c(data, 0, data.length); return CHECKSUM_SUPPLIER.get();
} }
public static int maskedCrc32c(byte[] data, int offset, int length) public static int maskedCrc32c(Checksum crc32c, byte[] data, int offset, int length)
{ {
final PureJavaCrc32C crc32c = new PureJavaCrc32C(); crc32c.reset();
crc32c.update(data, offset, length); crc32c.update(data, offset, length);
return mask(crc32c.getIntegerValue()); return mask((int) crc32c.getValue());
} }
/** /**

View File

@ -21,6 +21,7 @@ import java.nio.channels.ClosedChannelException;
import java.nio.channels.ReadableByteChannel; import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel; import java.nio.channels.WritableByteChannel;
import java.util.Arrays; import java.util.Arrays;
import java.util.zip.Checksum;
import org.xerial.snappy.pool.BufferPool; import org.xerial.snappy.pool.BufferPool;
import org.xerial.snappy.pool.DefaultPoolFactory; import org.xerial.snappy.pool.DefaultPoolFactory;
@ -40,6 +41,7 @@ public final class SnappyFramedInputStream
ReadableByteChannel ReadableByteChannel
{ {
private final Checksum crc32 = SnappyFramed.getCRC32C();
private final ReadableByteChannel rbc; private final ReadableByteChannel rbc;
private final ByteBuffer frameHeader; private final ByteBuffer frameHeader;
private final boolean verifyChecksums; private final boolean verifyChecksums;
@ -572,7 +574,7 @@ public final class SnappyFramedInputStream
} }
if (verifyChecksums) { if (verifyChecksums) {
final int actualCrc32c = SnappyFramed.maskedCrc32c(buffer, final int actualCrc32c = SnappyFramed.maskedCrc32c(crc32, buffer,
position, valid - position); position, valid - position);
if (frameData.checkSum != actualCrc32c) { if (frameData.checkSum != actualCrc32c) {
throw new IOException("Corrupt input: invalid checksum"); throw new IOException("Corrupt input: invalid checksum");

View File

@ -17,6 +17,7 @@ import java.nio.channels.Channels;
import java.nio.channels.ClosedChannelException; import java.nio.channels.ClosedChannelException;
import java.nio.channels.ReadableByteChannel; import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel; import java.nio.channels.WritableByteChannel;
import java.util.zip.Checksum;
import org.xerial.snappy.pool.BufferPool; import org.xerial.snappy.pool.BufferPool;
import org.xerial.snappy.pool.DefaultPoolFactory; import org.xerial.snappy.pool.DefaultPoolFactory;
@ -59,6 +60,7 @@ public final class SnappyFramedOutputStream
*/ */
public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d; public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d;
private final Checksum crc32 = SnappyFramed.getCRC32C();
private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order( private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order(
ByteOrder.LITTLE_ENDIAN); ByteOrder.LITTLE_ENDIAN);
private final BufferPool bufferPool; private final BufferPool bufferPool;
@ -503,7 +505,7 @@ public final class SnappyFramedOutputStream
final int length = buffer.remaining(); final int length = buffer.remaining();
// crc is based on the user supplied input data // crc is based on the user supplied input data
final int crc32c = maskedCrc32c(input, 0, length); final int crc32c = maskedCrc32c(crc32, input, 0, length);
directInputBuffer.clear(); directInputBuffer.clear();
directInputBuffer.put(buffer); directInputBuffer.put(buffer);

View File

@ -9,7 +9,6 @@ import static org.junit.Assert.fail;
import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -418,4 +417,9 @@ public class SnappyFramedStreamTest
assertEquals(random.length, length); assertEquals(random.length, length);
return random; return random;
} }
public static int maskedCrc32c(byte[] data)
{
return SnappyFramed.maskedCrc32c(new PureJavaCrc32C(), data, 0, data.length);
}
} }