Compare commits

..

2 Commits

Author SHA1 Message Date
comp500
a0da889a02 Optimise memory usage while computing Murmur2 2019-12-23 16:31:36 +00:00
comp500
432bb4e25f Fix Murmur2 hash implementation 2019-12-23 16:20:38 +00:00
2 changed files with 30 additions and 15 deletions

View File

@@ -74,13 +74,13 @@ public class Murmur2Lib {
int left = length - len_m; int left = length - len_m;
if (left != 0) { if (left != 0) {
if (left >= 3) { if (left >= 3) {
h ^= (int) data[length - 3] << 16; h ^= (int) data[length - (left - 2)] << 16;
} }
if (left >= 2) { if (left >= 2) {
h ^= (int) data[length - 2] << 8; h ^= (int) data[length - (left - 1)] << 8;
} }
if (left >= 1) { if (left >= 1) {
h ^= (int) data[length - 1]; h ^= data[length - left];
} }
h *= M_32; h *= M_32;
@@ -152,7 +152,7 @@ public class Murmur2Lib {
case 2: case 2:
h ^= (long) (data[tailStart + 1] & 0xff) << 8; h ^= (long) (data[tailStart + 1] & 0xff) << 8;
case 1: case 1:
h ^= (long) (data[tailStart] & 0xff); h ^= data[tailStart] & 0xff;
h *= M_64; h *= M_64;
} }

View File

@@ -10,7 +10,7 @@ class Murmur2Hasher : IHasher {
val tempBuffer = Buffer() val tempBuffer = Buffer()
override val hash: Hash by lazy(LazyThreadSafetyMode.NONE) { override val hash: Hash by lazy(LazyThreadSafetyMode.NONE) {
val data = computeNormalizedArray(internalBuffer.readByteArray()) val data = internalBuffer.readByteArray()
Murmur2Hash(Murmur2Lib.hash32(data, data.size, 1)) Murmur2Hash(Murmur2Lib.hash32(data, data.size, 1))
} }
@@ -19,27 +19,42 @@ class Murmur2Hasher : IHasher {
val out = delegate.read(tempBuffer, byteCount) val out = delegate.read(tempBuffer, byteCount)
if (out > -1) { if (out > -1) {
sink.write(tempBuffer.clone(), out) sink.write(tempBuffer.clone(), out)
internalBuffer.write(tempBuffer, out) computeNormalizedBufferFaster(tempBuffer, internalBuffer)
} }
return out return out
} }
// Credit to https://github.com/modmuss50/CAV2/blob/master/murmur.go // Credit to https://github.com/modmuss50/CAV2/blob/master/murmur.go
private fun computeNormalizedArray(input: ByteArray): ByteArray { // private fun computeNormalizedArray(input: ByteArray): ByteArray {
val output = ByteArray(input.size) // val output = ByteArray(input.size)
// var index = 0
// for (b in input) {
// when (b) {
// 9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
// else -> {
// output[index] = b
// index++
// }
// }
// }
// val outputTrimmed = ByteArray(index)
// System.arraycopy(output, 0, outputTrimmed, 0, index)
// return outputTrimmed
// }
private fun computeNormalizedBufferFaster(input: Buffer, output: Buffer) {
var index = 0 var index = 0
for (b in input) { val arr = input.readByteArray()
when (b.toInt()) { for (b in arr) {
9, 10, 13, 32 -> {} when (b) {
9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
else -> { else -> {
output[index] = b arr[index] = b
index++ index++
} }
} }
} }
val outputTrimmed = ByteArray(index) output.write(arr, 0, index)
System.arraycopy(output, 0, outputTrimmed, 0, index)
return outputTrimmed
} }
} }