tyler.durden
/
snappy-java
mirror of https://github.com/xerial/snappy-java.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Big Endian support to Pure Java Snappy (#254) 

* Add Big Endian support to Pure Java Snappy

* Support Big Endian. Separated Little and Big Endian methods for performance reasons

* Big Endian support - made a separate interface so no performance issues

* Big Endian support update - Remove trivial whitespace/tab changes which othewise come up in diff report

* Add newline to end of source for githib diff report

* Big Endian Support - simplified changes as peformance hit is not significant

* Big Endian Support - fix typo

* Fix typo

* Update SnappyLoader.java

Converted TAB char to spaces

* Revert back to original code

Co-authored-by: Russell Shaw <russell@Russells-MacBook-Pro.local>
This commit is contained in:
Russell Shaw 2020-10-20 21:03:58 -04:00 committed by GitHub
parent fd26e3914d
commit f368c0c7b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 46 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
src/main/java/org/xerial/snappy/SnappyLoader.java Executable file → Normal file
View File

44
src/main/java/org/xerial/snappy/pure/SnappyRawCompressor.java
View File

@ -12,6 +12,8 @@
* limitations under the License.
*/
package org.xerial.snappy.pure;
import java.nio.ByteOrder;
import java.util.Arrays;
import static org.xerial.snappy.pure.SnappyConstants.COPY_1_BYTE_OFFSET;
@ -20,6 +22,9 @@ import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_INT;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_LONG;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_SHORT;
import static org.xerial.snappy.pure.UnsafeUtil.UNSAFE;
import static java.lang.Integer.reverseBytes;
import static java.lang.Long.reverseBytes;
import static java.lang.Short.reverseBytes;
public final class SnappyRawCompressor
{
@ -40,8 +45,25 @@ public final class SnappyRawCompressor
private static final int MAX_HASH_TABLE_BITS = 14;
public static final int MAX_HASH_TABLE_SIZE = 1 << MAX_HASH_TABLE_BITS;
private static final ByteOrder byteOrder = ByteOrder.nativeOrder();
private SnappyRawCompressor() {}
private static int littleEndian(int i)
{
return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
}
private static long littleEndian(long i)
{
return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
}
private static short littleEndian(short i)
{
return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
}
public static int maxCompressedLength(int sourceLength)
{
// Compressed data can be defined as:
@ -138,7 +160,7 @@ public final class SnappyRawCompressor
long candidateIndex = 0;
for (input += 1; input + (skip >>> 5) <= fastInputLimit; input += ((skip++) >>> 5)) {
// hash the 4 bytes starting at the input pointer
int currentInt = UNSAFE.getInt(inputBase, input);
int currentInt = littleEndian(UNSAFE.getInt(inputBase, input));
int hash = hashBytes(currentInt, shift);
// get the position of a 4 bytes sequence with the same hash
@ -151,7 +173,7 @@ public final class SnappyRawCompressor
// if the 4 byte sequence a the candidate index matches the sequence at the
// current position, proceed to the next phase
if (currentInt == UNSAFE.getInt(inputBase, candidateIndex)) {
if (currentInt == littleEndian(UNSAFE.getInt(inputBase, candidateIndex))) {
break;
}
}
@ -201,7 +223,7 @@ public final class SnappyRawCompressor
// We could immediately start working at input now, but to improve
// compression we first update table[Hash(ip - 1, ...)].
long longValue = UNSAFE.getLong(inputBase, input - 1);
long longValue = littleEndian((UNSAFE.getLong(inputBase, input - 1)));
int prevInt = (int) longValue;
inputBytes = (int) (longValue >>> 8);
@ -214,7 +236,7 @@ public final class SnappyRawCompressor
candidateIndex = blockAddress + (table[curHash] & 0xFFFF);
table[curHash] = (short) (input - blockAddress);
} while (inputBytes == UNSAFE.getInt(inputBase, candidateIndex));
} while (inputBytes == littleEndian(UNSAFE.getInt(inputBase, candidateIndex)));
nextEmitAddress = input;
}
@ -236,7 +258,7 @@ public final class SnappyRawCompressor
// first, compare long at a time
while (current < matchLimit - (SIZE_OF_LONG - 1)) {
long diff = UNSAFE.getLong(inputBase, matchStart) ^ UNSAFE.getLong(inputBase, current);
long diff = littleEndian(UNSAFE.getLong(inputBase, matchStart)) ^ littleEndian(UNSAFE.getLong(inputBase, current));
if (diff != 0) {
current += Long.numberOfTrailingZeros(diff) >> 3;
return (int) (current - start);
@ -246,12 +268,12 @@ public final class SnappyRawCompressor
matchStart += SIZE_OF_LONG;
}
if (current < matchLimit - (SIZE_OF_INT - 1) && UNSAFE.getInt(inputBase, matchStart) == UNSAFE.getInt(inputBase, current)) {
if (current < matchLimit - (SIZE_OF_INT - 1) && littleEndian(UNSAFE.getInt(inputBase, matchStart)) == littleEndian(UNSAFE.getInt(inputBase, current))) {
current += SIZE_OF_INT;
matchStart += SIZE_OF_INT;
}
if (current < matchLimit - (SIZE_OF_SHORT - 1) && UNSAFE.getShort(inputBase, matchStart) == UNSAFE.getShort(inputBase, current)) {
if (current < matchLimit - (SIZE_OF_SHORT - 1) && littleEndian(UNSAFE.getShort(inputBase, matchStart)) == littleEndian(UNSAFE.getShort(inputBase, current))) {
current += SIZE_OF_SHORT;
matchStart += SIZE_OF_SHORT;
}
@ -289,7 +311,7 @@ public final class SnappyRawCompressor
bytes = 4;
}
// System is assumed to be little endian, so low bytes will be zero for the smaller numbers
UNSAFE.putInt(outputBase, output, n);
UNSAFE.putInt(outputBase, output, littleEndian(n));
output += bytes;
}
return output;
@ -314,7 +336,7 @@ public final class SnappyRawCompressor
// Emit 64 byte copies but make sure to keep at least four bytes reserved
while (matchLength >= 68) {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((64 - 1) << 2)));
UNSAFE.putShort(outputBase, output, (short) offset);
UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
matchLength -= 64;
}
@ -323,7 +345,7 @@ public final class SnappyRawCompressor
// length < 68
if (matchLength > 64) {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((60 - 1) << 2)));
UNSAFE.putShort(outputBase, output, (short) offset);
UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
matchLength -= 60;
}
@ -336,7 +358,7 @@ public final class SnappyRawCompressor
}
else {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((matchLength - 1) << 2)));
UNSAFE.putShort(outputBase, output, (short) offset);
UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
}
return output;

11
src/main/java/org/xerial/snappy/pure/SnappyRawDecompressor.java
View File

@ -13,6 +13,8 @@
*/
package org.xerial.snappy.pure;
import java.nio.ByteOrder;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyErrorCode;
@ -20,6 +22,7 @@ import static org.xerial.snappy.pure.SnappyConstants.LITERAL;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_INT;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_LONG;
import static org.xerial.snappy.pure.UnsafeUtil.UNSAFE;
import static java.lang.Integer.reverseBytes;
public final class SnappyRawDecompressor
{
@ -28,6 +31,12 @@ public final class SnappyRawDecompressor
private SnappyRawDecompressor() {}
private static final ByteOrder byteOrder = ByteOrder.nativeOrder();
private static int littleEndian(int i) {
return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
}
public static int getUncompressedLength(Object compressed, long compressedAddress, long compressedLimit)
{
return readUncompressedLength(compressed, compressedAddress, compressedLimit)[0];
@ -89,7 +98,7 @@ public final class SnappyRawDecompressor
int trailerBytes = entry >>> 11;
int trailer = 0;
if (input + SIZE_OF_INT < inputLimit) {
trailer = UNSAFE.getInt(inputBase, input) & wordmask[trailerBytes];
trailer = littleEndian(UNSAFE.getInt(inputBase, input)) & wordmask[trailerBytes];
}
else {
if (input + trailerBytes > inputLimit) {

9
src/main/java/org/xerial/snappy/pure/UnsafeUtil.java
View File

@ -19,9 +19,6 @@ import sun.misc.Unsafe;
import java.lang.reflect.Field;
import java.nio.Buffer;
import java.nio.ByteOrder;
import static java.lang.String.format;
final class UnsafeUtil
{
@ -33,10 +30,6 @@ final class UnsafeUtil
}
static {
ByteOrder order = ByteOrder.nativeOrder();
if (!order.equals(ByteOrder.LITTLE_ENDIAN)) {
throw new SnappyError(SnappyErrorCode.UNSUPPORTED_PLATFORM, format("pure-java snappy requires a little endian platform (found %s)", order));
}
try {
Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
@ -66,4 +59,6 @@ final class UnsafeUtil
throw new RuntimeException(e);
}
}
}