From b079ae87423645f2da0636713e4afec9e62a64ea Mon Sep 17 00:00:00 2001 From: a1denvalu3 <43107113+a1denvalu3@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:00:03 +0200 Subject: [PATCH] fix(sync): resolve hash alignment and early break bugs on GCS filter trimming (#708) * fix(sync): resolve hash alignment and early break bugs on GCS filter trimming * fix(sync): filter out zero bucket and deduplicate buckets before encoding * fix(sync): preserve zero-bucket mapping by clamping 0 to 1 during encoding and decoding --- .../com/bitchat/android/sync/GCSFilter.kt | 30 ++++--- .../bitchat/android/sync/GossipSyncManager.kt | 11 +-- .../com/bitchat/android/sync/GCSFilterTest.kt | 86 +++++++++++++++++++ 3 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 app/src/test/kotlin/com/bitchat/android/sync/GCSFilterTest.kt diff --git a/app/src/main/java/com/bitchat/android/sync/GCSFilter.kt b/app/src/main/java/com/bitchat/android/sync/GCSFilter.kt index 3cc64c58..212def6a 100644 --- a/app/src/main/java/com/bitchat/android/sync/GCSFilter.kt +++ b/app/src/main/java/com/bitchat/android/sync/GCSFilter.kt @@ -43,21 +43,29 @@ object GCSFilter { targetFpr: Double ): Params { val p = deriveP(targetFpr) - var nCap = estimateMaxElementsForSize(maxBytes, p) - val n = ids.size.coerceAtMost(nCap) - val selected = ids.take(n) - // Map to [0, M) - val m = (n.toLong() shl p) - val mapped = selected.map { id -> (h64(id) % m) }.sorted() + val nCap = estimateMaxElementsForSize(maxBytes, p) + var trimmedN = ids.size.coerceAtMost(nCap) + + var finalM = (trimmedN.toLong() shl p).coerceAtLeast(1L) + var selected = ids.take(trimmedN) + var mapped = selected.map { id -> + val v = h64(id) % finalM + if (v == 0L) 1L else v + }.distinct().sorted() var encoded = encode(mapped, p) + // If estimate was too optimistic, trim until it fits - var trimmedN = n while (encoded.size > maxBytes && trimmedN > 0) { trimmedN = (trimmedN * 9) / 10 // drop 10% - val mapped2 = mapped.take(trimmedN) - encoded = encode(mapped2, p) + finalM = (trimmedN.toLong() shl p).coerceAtLeast(1L) + selected = ids.take(trimmedN) + mapped = selected.map { id -> + val v = h64(id) % finalM + if (v == 0L) 1L else v + }.distinct().sorted() + encoded = encode(mapped, p) } - val finalM = (trimmedN.toLong() shl p) + return Params(p = p, m = finalM, data = encoded) } @@ -96,7 +104,7 @@ object GCSFilter { return false } - private fun h64(id16: ByteArray): Long { + internal fun h64(id16: ByteArray): Long { val md = MessageDigest.getInstance("SHA-256") md.update(id16) val d = md.digest() diff --git a/app/src/main/java/com/bitchat/android/sync/GossipSyncManager.kt b/app/src/main/java/com/bitchat/android/sync/GossipSyncManager.kt index 6e29aa79..5c786365 100644 --- a/app/src/main/java/com/bitchat/android/sync/GossipSyncManager.kt +++ b/app/src/main/java/com/bitchat/android/sync/GossipSyncManager.kt @@ -169,14 +169,9 @@ class GossipSyncManager( // Decode GCS into sorted set for membership checks val sorted = GCSFilter.decodeToSortedSet(request.p, request.m, request.data) fun mightContain(id: ByteArray): Boolean { - val v = (GCSFilter.run { - // reuse hashing method from GCSFilter - val md = java.security.MessageDigest.getInstance("SHA-256"); - md.update(id); val d = md.digest(); - var x = 0L; for (i in 0 until 8) { x = (x shl 8) or (d[i].toLong() and 0xFF) } - (x and 0x7fff_ffff_ffff_ffffL) % request.m - }) - return GCSFilter.contains(sorted, v) + val v = GCSFilter.h64(id) % request.m + val nonZeroV = if (v == 0L) 1L else v + return GCSFilter.contains(sorted, nonZeroV) } // 1) Announcements: send latest per peerID if remote doesn't have them diff --git a/app/src/test/kotlin/com/bitchat/android/sync/GCSFilterTest.kt b/app/src/test/kotlin/com/bitchat/android/sync/GCSFilterTest.kt new file mode 100644 index 00000000..3a9bfb62 --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/sync/GCSFilterTest.kt @@ -0,0 +1,86 @@ +package com.bitchat.android.sync + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.Random + +class GCSFilterTest { + + @Test + fun testGCSFilterBasic() { + val random = Random(42) + val ids = List(20) { + val bytes = ByteArray(16) + random.nextBytes(bytes) + bytes + } + + // Build filter with plenty of bytes (no trimming) + val params = GCSFilter.buildFilter(ids, maxBytes = 400, targetFpr = 0.01) + val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data) + + for (id in ids) { + val v = GCSFilter.h64(id) % params.m + val nonZeroV = if (v == 0L) 1L else v + assertTrue("Filter should contain all encoded IDs", GCSFilter.contains(sorted, nonZeroV)) + } + } + + @Test + fun testGCSFilterWithTrimming() { + val random = Random(42) + // 50 IDs + val ids = List(50) { + val bytes = ByteArray(16) + random.nextBytes(bytes) + bytes + } + + // Force trimming by setting maxBytes to a very small value (e.g., 20 bytes) + val maxBytes = 20 + val params = GCSFilter.buildFilter(ids, maxBytes = maxBytes, targetFpr = 0.01) + + // Ensure some trimming actually happened + assertTrue("Params data size should be <= maxBytes", params.data.size <= maxBytes) + + val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data) + + // Let's verify that the first trimmedN elements in ids are all matched + val trimmedN = (params.m ushr params.p).toInt() + assertTrue("At least some elements should have been encoded", trimmedN > 0) + + val retainedIds = ids.take(trimmedN) + for (id in retainedIds) { + val v = GCSFilter.h64(id) % params.m + val nonZeroV = if (v == 0L) 1L else v + assertTrue("Retained ID should be found in filter", GCSFilter.contains(sorted, nonZeroV)) + } + } + + @Test + fun testGCSFilterHandlesCollisionsAndZeroBucket() { + val random = Random(42) + // Generate a large number of IDs to guarantee collisions (mapping to the same bucket) and some zero-bucket mapping + val ids = List(200) { + val bytes = ByteArray(16) + random.nextBytes(bytes) + bytes + } + + // Build GCS filter - this should complete successfully without throwing repeat count exceptions or negative-count crashes + val params = GCSFilter.buildFilter(ids, maxBytes = 100, targetFpr = 0.05) + val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data) + + // Verify elements are successfully stored and found (including those mapping to 0) + var foundCount = 0 + for (id in ids) { + val v = GCSFilter.h64(id) % params.m + val nonZeroV = if (v == 0L) 1L else v + if (GCSFilter.contains(sorted, nonZeroV)) { + foundCount++ + } + } + assertTrue("Should successfully decode and find elements after deduplication", foundCount > 0) + } +}