fix(sync): resolve hash alignment and early break bugs on GCS filter trimming (#708)

* fix(sync): resolve hash alignment and early break bugs on GCS filter trimming

* fix(sync): filter out zero bucket and deduplicate buckets before encoding

* fix(sync): preserve zero-bucket mapping by clamping 0 to 1 during encoding and decoding

app/src/main/java/com/bitchat/android/sync/GCSFilter.kt

@@ -43,21 +43,29 @@ object GCSFilter {
         targetFpr: Double
     ): Params {
         val p = deriveP(targetFpr)
-        var nCap = estimateMaxElementsForSize(maxBytes, p)
-        val n = ids.size.coerceAtMost(nCap)
-        val selected = ids.take(n)
-        // Map to [0, M)
-        val m = (n.toLong() shl p)
-        val mapped = selected.map { id -> (h64(id) % m) }.sorted()
+        val nCap = estimateMaxElementsForSize(maxBytes, p)
+        var trimmedN = ids.size.coerceAtMost(nCap)
+
+        var finalM = (trimmedN.toLong() shl p).coerceAtLeast(1L)
+        var selected = ids.take(trimmedN)
+        var mapped = selected.map { id ->
+            val v = h64(id) % finalM
+            if (v == 0L) 1L else v
+        }.distinct().sorted()
         var encoded = encode(mapped, p)
+
         // If estimate was too optimistic, trim until it fits
-        var trimmedN = n
         while (encoded.size > maxBytes && trimmedN > 0) {
             trimmedN = (trimmedN * 9) / 10 // drop 10%
-            val mapped2 = mapped.take(trimmedN)
-            encoded = encode(mapped2, p)
+            finalM = (trimmedN.toLong() shl p).coerceAtLeast(1L)
+            selected = ids.take(trimmedN)
+            mapped = selected.map { id ->
+                val v = h64(id) % finalM
+                if (v == 0L) 1L else v
+            }.distinct().sorted()
+            encoded = encode(mapped, p)
         }
-        val finalM = (trimmedN.toLong() shl p)
+
         return Params(p = p, m = finalM, data = encoded)
     }
 
@@ -96,7 +104,7 @@ object GCSFilter {
         return false
     }
 
-    private fun h64(id16: ByteArray): Long {
+    internal fun h64(id16: ByteArray): Long {
         val md = MessageDigest.getInstance("SHA-256")
         md.update(id16)
         val d = md.digest()

app/src/main/java/com/bitchat/android/sync/GossipSyncManager.kt

@@ -169,14 +169,9 @@ class GossipSyncManager(
         // Decode GCS into sorted set for membership checks
         val sorted = GCSFilter.decodeToSortedSet(request.p, request.m, request.data)
         fun mightContain(id: ByteArray): Boolean {
-            val v = (GCSFilter.run {
-                // reuse hashing method from GCSFilter
-                val md = java.security.MessageDigest.getInstance("SHA-256");
-                md.update(id); val d = md.digest();
-                var x = 0L; for (i in 0 until 8) { x = (x shl 8) or (d[i].toLong() and 0xFF) }
-                (x and 0x7fff_ffff_ffff_ffffL) % request.m
-            })
-            return GCSFilter.contains(sorted, v)
+            val v = GCSFilter.h64(id) % request.m
+            val nonZeroV = if (v == 0L) 1L else v
+            return GCSFilter.contains(sorted, nonZeroV)
         }
 
         // 1) Announcements: send latest per peerID if remote doesn't have them

app/src/test/kotlin/com/bitchat/android/sync/GCSFilterTest.kt

@@ -0,0 +1,86 @@
+package com.bitchat.android.sync
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import java.util.Random
+
+class GCSFilterTest {
+
+    @Test
+    fun testGCSFilterBasic() {
+        val random = Random(42)
+        val ids = List(20) {
+            val bytes = ByteArray(16)
+            random.nextBytes(bytes)
+            bytes
+        }
+
+        // Build filter with plenty of bytes (no trimming)
+        val params = GCSFilter.buildFilter(ids, maxBytes = 400, targetFpr = 0.01)
+        val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data)
+
+        for (id in ids) {
+            val v = GCSFilter.h64(id) % params.m
+            val nonZeroV = if (v == 0L) 1L else v
+            assertTrue("Filter should contain all encoded IDs", GCSFilter.contains(sorted, nonZeroV))
+        }
+    }
+
+    @Test
+    fun testGCSFilterWithTrimming() {
+        val random = Random(42)
+        // 50 IDs
+        val ids = List(50) {
+            val bytes = ByteArray(16)
+            random.nextBytes(bytes)
+            bytes
+        }
+
+        // Force trimming by setting maxBytes to a very small value (e.g., 20 bytes)
+        val maxBytes = 20
+        val params = GCSFilter.buildFilter(ids, maxBytes = maxBytes, targetFpr = 0.01)
+        
+        // Ensure some trimming actually happened
+        assertTrue("Params data size should be <= maxBytes", params.data.size <= maxBytes)
+        
+        val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data)
+
+        // Let's verify that the first trimmedN elements in ids are all matched
+        val trimmedN = (params.m ushr params.p).toInt()
+        assertTrue("At least some elements should have been encoded", trimmedN > 0)
+        
+        val retainedIds = ids.take(trimmedN)
+        for (id in retainedIds) {
+            val v = GCSFilter.h64(id) % params.m
+            val nonZeroV = if (v == 0L) 1L else v
+            assertTrue("Retained ID should be found in filter", GCSFilter.contains(sorted, nonZeroV))
+        }
+    }
+
+    @Test
+    fun testGCSFilterHandlesCollisionsAndZeroBucket() {
+        val random = Random(42)
+        // Generate a large number of IDs to guarantee collisions (mapping to the same bucket) and some zero-bucket mapping
+        val ids = List(200) {
+            val bytes = ByteArray(16)
+            random.nextBytes(bytes)
+            bytes
+        }
+
+        // Build GCS filter - this should complete successfully without throwing repeat count exceptions or negative-count crashes
+        val params = GCSFilter.buildFilter(ids, maxBytes = 100, targetFpr = 0.05)
+        val sorted = GCSFilter.decodeToSortedSet(params.p, params.m, params.data)
+
+        // Verify elements are successfully stored and found (including those mapping to 0)
+        var foundCount = 0
+        for (id in ids) {
+            val v = GCSFilter.h64(id) % params.m
+            val nonZeroV = if (v == 0L) 1L else v
+            if (GCSFilter.contains(sorted, nonZeroV)) {
+                foundCount++
+            }
+        }
+        assertTrue("Should successfully decode and find elements after deduplication", foundCount > 0)
+    }
+}