Compare commits

..

2 Commits

Author SHA1 Message Date
comp500
a0da889a02 Optimise memory usage while computing Murmur2 2019-12-23 16:31:36 +00:00
comp500
432bb4e25f Fix Murmur2 hash implementation 2019-12-23 16:20:38 +00:00
2 changed files with 30 additions and 15 deletions

View File

@@ -74,13 +74,13 @@ public class Murmur2Lib {
int left = length - len_m;
if (left != 0) {
if (left >= 3) {
h ^= (int) data[length - 3] << 16;
h ^= (int) data[length - (left - 2)] << 16;
}
if (left >= 2) {
h ^= (int) data[length - 2] << 8;
h ^= (int) data[length - (left - 1)] << 8;
}
if (left >= 1) {
h ^= (int) data[length - 1];
h ^= data[length - left];
}
h *= M_32;
@@ -152,7 +152,7 @@ public class Murmur2Lib {
case 2:
h ^= (long) (data[tailStart + 1] & 0xff) << 8;
case 1:
h ^= (long) (data[tailStart] & 0xff);
h ^= data[tailStart] & 0xff;
h *= M_64;
}

View File

@@ -10,7 +10,7 @@ class Murmur2Hasher : IHasher {
val tempBuffer = Buffer()
override val hash: Hash by lazy(LazyThreadSafetyMode.NONE) {
val data = computeNormalizedArray(internalBuffer.readByteArray())
val data = internalBuffer.readByteArray()
Murmur2Hash(Murmur2Lib.hash32(data, data.size, 1))
}
@@ -19,27 +19,42 @@ class Murmur2Hasher : IHasher {
val out = delegate.read(tempBuffer, byteCount)
if (out > -1) {
sink.write(tempBuffer.clone(), out)
internalBuffer.write(tempBuffer, out)
computeNormalizedBufferFaster(tempBuffer, internalBuffer)
}
return out
}
// Credit to https://github.com/modmuss50/CAV2/blob/master/murmur.go
private fun computeNormalizedArray(input: ByteArray): ByteArray {
val output = ByteArray(input.size)
// private fun computeNormalizedArray(input: ByteArray): ByteArray {
// val output = ByteArray(input.size)
// var index = 0
// for (b in input) {
// when (b) {
// 9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
// else -> {
// output[index] = b
// index++
// }
// }
// }
// val outputTrimmed = ByteArray(index)
// System.arraycopy(output, 0, outputTrimmed, 0, index)
// return outputTrimmed
// }
private fun computeNormalizedBufferFaster(input: Buffer, output: Buffer) {
var index = 0
for (b in input) {
when (b.toInt()) {
9, 10, 13, 32 -> {}
val arr = input.readByteArray()
for (b in arr) {
when (b) {
9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
else -> {
output[index] = b
arr[index] = b
index++
}
}
}
val outputTrimmed = ByteArray(index)
System.arraycopy(output, 0, outputTrimmed, 0, index)
return outputTrimmed
output.write(arr, 0, index)
}
}