Align REQUEST_SYNC filters with Android

2026-07-24 22:25:18 +00:00 · 2026-06-25 23:09:35 +02:00
parent f688e529f6
commit 842d8f2cd3
5 changed files with 295 additions and 34 deletions
@@ -4,6 +4,9 @@ import Foundation
 //  - 0x01: P (uint8) — Golomb-Rice parameter
 //  - 0x02: M (uint32, big-endian) — hash range (N * 2^P)
 //  - 0x03: data (opaque) — GR bitstream bytes (MSB-first)
+//  - 0x04: wanted types — raw MessageType bytes
+//  - 0x05: minimum timestamp (uint64, big-endian) — epoch milliseconds
+//  - 0x06: fragment id filter (utf8)
 struct RequestSyncPacket {
    let p: Int
    let m: UInt32
@@ -167,6 +167,16 @@ final class GossipSyncManager {
        return packet.timestamp >= cutoffMs
    }

+    private func normalizedSyncTypes(_ types: SyncTypeFlags?) -> SyncTypeFlags {
+        guard let types, !types.isEmpty else { return .publicMessages }
+        return types
+    }
+
+    private func isAtOrAfterMinimumTimestamp(_ packet: BitchatPacket, sinceTimestamp: UInt64?) -> Bool {
+        guard let sinceTimestamp else { return true }
+        return packet.timestamp >= sinceTimestamp
+    }
+
    private func _onPublicPacketSeen(_ packet: BitchatPacket) {
        guard let messageType = MessageType(rawValue: packet.type) else { return }
        let isBroadcastRecipient: Bool = {
@@ -218,8 +228,8 @@ final class GossipSyncManager {
        }
    }

-    private func sendRequestSync(for types: SyncTypeFlags) {
-        let payload = buildGcsPayload(for: types)
+    func sendRequestSync(for types: SyncTypeFlags? = nil, sinceTimestamp: UInt64? = nil) {
+        let payload = buildGcsPayload(for: types, sinceTimestamp: sinceTimestamp)
        let pkt = BitchatPacket(
            type: MessageType.requestSync.rawValue,
            senderID: Data(hexString: myPeerID.id) ?? Data(),
@@ -233,11 +243,11 @@ final class GossipSyncManager {
        delegate?.sendPacket(signed)
    }

-    private func sendRequestSync(to peerID: PeerID, types: SyncTypeFlags) {
+    func sendRequestSync(to peerID: PeerID, types: SyncTypeFlags? = nil, sinceTimestamp: UInt64? = nil) {
        // Register the request for RSR validation
        requestSyncManager.registerRequest(to: peerID)
        
-        let payload = buildGcsPayload(for: types)
+        let payload = buildGcsPayload(for: types, sinceTimestamp: sinceTimestamp)
        var recipient = Data()
        var temp = peerID.id
        while temp.count >= 2 && recipient.count < 8 {
@@ -265,7 +275,8 @@ final class GossipSyncManager {
    }

    private func _handleRequestSync(from peerID: PeerID, request: RequestSyncPacket) {
-        let requestedTypes = (request.types ?? .publicMessages)
+        let requestedTypes = normalizedSyncTypes(request.types)
+        let sinceTimestamp = request.sinceTimestamp
        // Decode GCS into sorted set and prepare membership checker
        let sorted = GCSFilter.decodeToSortedSet(p: request.p, m: request.m, data: request.data)
        func mightContain(_ id: Data) -> Bool {
@@ -276,7 +287,7 @@ final class GossipSyncManager {
        if requestedTypes.contains(.announce) {
            for (_, pair) in latestAnnouncementByPeer {
                let (idHex, pkt) = pair
-                guard isPacketFresh(pkt) else { continue }
+                guard isPacketFresh(pkt), isAtOrAfterMinimumTimestamp(pkt, sinceTimestamp: sinceTimestamp) else { continue }
                let idBytes = Data(hexString: idHex) ?? Data()
                if !mightContain(idBytes) {
                    var toSend = pkt
@@ -290,6 +301,7 @@ final class GossipSyncManager {
        if requestedTypes.contains(.message) {
            let toSendMsgs = messages.allPackets(isFresh: isPacketFresh)
            for pkt in toSendMsgs {
+                guard isAtOrAfterMinimumTimestamp(pkt, sinceTimestamp: sinceTimestamp) else { continue }
                let idBytes = PacketIdUtil.computeId(pkt)
                if !mightContain(idBytes) {
                    var toSend = pkt
@@ -303,6 +315,7 @@ final class GossipSyncManager {
        if requestedTypes.contains(.fragment) {
            let frags = fragments.allPackets(isFresh: isPacketFresh)
            for pkt in frags {
+                guard isAtOrAfterMinimumTimestamp(pkt, sinceTimestamp: sinceTimestamp) else { continue }
                let idBytes = PacketIdUtil.computeId(pkt)
                if !mightContain(idBytes) {
                    var toSend = pkt
@@ -316,6 +329,7 @@ final class GossipSyncManager {
        if requestedTypes.contains(.fileTransfer) {
            let files = fileTransfers.allPackets(isFresh: isPacketFresh)
            for pkt in files {
+                guard isAtOrAfterMinimumTimestamp(pkt, sinceTimestamp: sinceTimestamp) else { continue }
                let idBytes = PacketIdUtil.computeId(pkt)
                if !mightContain(idBytes) {
                    var toSend = pkt
@@ -328,25 +342,33 @@ final class GossipSyncManager {
    }

    // Build REQUEST_SYNC payload using current candidates and GCS params
-    private func buildGcsPayload(for types: SyncTypeFlags) -> Data {
+    private func buildGcsPayload(for types: SyncTypeFlags?, sinceTimestamp: UInt64? = nil) -> Data {
+        let requestedTypes = normalizedSyncTypes(types)
+        let encodedTypes = types.flatMap { $0.isEmpty ? nil : $0 }
        var candidates: [BitchatPacket] = []
-        if types.contains(.announce) {
-            for (_, pair) in latestAnnouncementByPeer where isPacketFresh(pair.packet) {
+        if requestedTypes.contains(.announce) {
+            for (_, pair) in latestAnnouncementByPeer where isPacketFresh(pair.packet) && isAtOrAfterMinimumTimestamp(pair.packet, sinceTimestamp: sinceTimestamp) {
                candidates.append(pair.packet)
            }
        }
-        if types.contains(.message) {
-            candidates.append(contentsOf: messages.allPackets(isFresh: isPacketFresh))
+        if requestedTypes.contains(.message) {
+            candidates.append(contentsOf: messages.allPackets(isFresh: isPacketFresh).filter {
+                isAtOrAfterMinimumTimestamp($0, sinceTimestamp: sinceTimestamp)
+            })
        }
-        if types.contains(.fragment) {
-            candidates.append(contentsOf: fragments.allPackets(isFresh: isPacketFresh))
+        if requestedTypes.contains(.fragment) {
+            candidates.append(contentsOf: fragments.allPackets(isFresh: isPacketFresh).filter {
+                isAtOrAfterMinimumTimestamp($0, sinceTimestamp: sinceTimestamp)
+            })
        }
-        if types.contains(.fileTransfer) {
-            candidates.append(contentsOf: fileTransfers.allPackets(isFresh: isPacketFresh))
+        if requestedTypes.contains(.fileTransfer) {
+            candidates.append(contentsOf: fileTransfers.allPackets(isFresh: isPacketFresh).filter {
+                isAtOrAfterMinimumTimestamp($0, sinceTimestamp: sinceTimestamp)
+            })
        }
        if candidates.isEmpty {
            let p = GCSFilter.deriveP(targetFpr: config.gcsTargetFpr)
-            let req = RequestSyncPacket(p: p, m: 1, data: Data(), types: types)
+            let req = RequestSyncPacket(p: p, m: 1, data: Data(), types: encodedTypes, sinceTimestamp: sinceTimestamp)
            return req.encode()
        }

@@ -356,21 +378,21 @@ final class GossipSyncManager {
        let p = GCSFilter.deriveP(targetFpr: config.gcsTargetFpr)
        let nMax = GCSFilter.estimateMaxElements(sizeBytes: config.gcsMaxBytes, p: p)
        let cap: Int
-        if types == .fragment {
+        if requestedTypes == .fragment {
            cap = max(1, config.fragmentCapacity)
-        } else if types == .fileTransfer {
+        } else if requestedTypes == .fileTransfer {
            cap = max(1, config.fileTransferCapacity)
        } else {
            cap = max(1, config.seenCapacity)
        }
        let takeN = min(candidates.count, min(nMax, cap))
        if takeN <= 0 {
-            let req = RequestSyncPacket(p: p, m: 1, data: Data(), types: types)
+            let req = RequestSyncPacket(p: p, m: 1, data: Data(), types: encodedTypes, sinceTimestamp: sinceTimestamp)
            return req.encode()
        }
        let ids: [Data] = candidates.prefix(takeN).map { PacketIdUtil.computeId($0) }
        let params = GCSFilter.buildFilter(ids: ids, maxBytes: config.gcsMaxBytes, targetFpr: config.gcsTargetFpr)
-        let req = RequestSyncPacket(p: params.p, m: params.m, data: params.data, types: types)
+        let req = RequestSyncPacket(p: params.p, m: params.m, data: params.data, types: encodedTypes, sinceTimestamp: sinceTimestamp)
        return req.encode()
    }

@@ -461,5 +483,11 @@ extension GossipSyncManager {
            messages.allPackets { _ in true }.filter { PeerID(hexData: $0.senderID) == peerID }.count
        }
    }
+
+    func _buildGcsPayloadSynchronously(for types: SyncTypeFlags? = nil, sinceTimestamp: UInt64? = nil) -> Data {
+        queue.sync {
+            buildGcsPayload(for: types, sinceTimestamp: sinceTimestamp)
+        }
+    }
 }
 #endif
@@ -1,8 +1,8 @@
 import BitFoundation
 import Foundation

-/// Bitfield describing which message types are covered by a REQUEST_SYNC round.
-/// Matches the Android mapping (bit index -> message type).
+/// Internal bitfield describing which message types are covered by a REQUEST_SYNC round.
+/// The wire TLV uses raw `MessageType` bytes, matching Android's wantedTypes format.
 struct SyncTypeFlags: OptionSet {
    let rawValue: UInt64

@@ -80,26 +80,37 @@ struct SyncTypeFlags: OptionSet {
    }

    func toData() -> Data? {
-        guard rawValue != 0 else { return nil }
-        var value = rawValue
-        var bytes: [UInt8] = []
-        while value > 0 && bytes.count < 8 {
-            bytes.append(UInt8(value & 0xFF))
-            value >>= 8
-        }
-        while let last = bytes.last, last == 0 {
-            bytes.removeLast()
-        }
-        guard !bytes.isEmpty, bytes.count <= 8 else { return nil }
+        let bytes = toMessageTypes().map(\.rawValue)
+        guard !bytes.isEmpty else { return nil }
        return Data(bytes)
    }

    static func decode(_ data: Data) -> SyncTypeFlags? {
+        guard !data.isEmpty else { return nil }
+
+        // Prior experimental iOS builds encoded announce+message as the
+        // bitfield byte 0x03. Android's upgraded wire format uses the same
+        // value for LEAVE, which this sync manager does not store, so prefer
+        // the legacy interpretation for the single-byte ambiguous case.
+        if data.count == 1 && data[0] == 0x03 {
+            return .publicMessages
+        }
+
+        let types = data.compactMap { MessageType(rawValue: $0) }
+        if !types.isEmpty {
+            return SyncTypeFlags(messageTypes: types)
+        }
+
+        return decodeLegacyBitfield(data)
+    }
+
+    private static func decodeLegacyBitfield(_ data: Data) -> SyncTypeFlags? {
        guard (1...8).contains(data.count) else { return nil }
        var raw: UInt64 = 0
        for (index, byte) in data.enumerated() {
            raw |= UInt64(byte) << UInt64(index * 8)
        }
-        return SyncTypeFlags(rawValue: raw)
+        let flags = SyncTypeFlags(rawValue: raw)
+        return flags.isEmpty ? nil : flags
    }
 }
@@ -279,6 +279,107 @@ struct GossipSyncManagerTests {
        #expect(sentPackets.count == 1)
        #expect(sentPackets[0].type == MessageType.fragment.rawValue)
    }
+
+    @Test func handleRequestSyncHonorsSinceTimestamp() async throws {
+        var config = GossipSyncManager.Config()
+        config.seenCapacity = 5
+        config.fragmentCapacity = 0
+        config.fileTransferCapacity = 0
+        config.messageSyncIntervalSeconds = 0
+        config.fragmentSyncIntervalSeconds = 0
+        config.fileTransferSyncIntervalSeconds = 0
+
+        let requestSyncManager = RequestSyncManager()
+        let manager = GossipSyncManager(myPeerID: myPeerID, config: config, requestSyncManager: requestSyncManager)
+        let delegate = RecordingDelegate()
+        manager.delegate = delegate
+
+        let sender = try #require(Data(hexString: "aabbccddeeff0011"))
+        let threshold = UInt64(Date().timeIntervalSince1970 * 1000)
+
+        let oldMessage = BitchatPacket(
+            type: MessageType.message.rawValue,
+            senderID: sender,
+            recipientID: nil,
+            timestamp: threshold - 1,
+            payload: Data([0x10]),
+            signature: nil,
+            ttl: 1
+        )
+
+        let freshMessage = BitchatPacket(
+            type: MessageType.message.rawValue,
+            senderID: sender,
+            recipientID: nil,
+            timestamp: threshold,
+            payload: Data([0x20]),
+            signature: nil,
+            ttl: 1
+        )
+
+        manager.onPublicPacketSeen(oldMessage)
+        manager.onPublicPacketSeen(freshMessage)
+
+        let peer = PeerID(str: "FFFFFFFFFFFFFFFF")
+        let request = RequestSyncPacket(p: 4, m: 1, data: Data(), types: .message, sinceTimestamp: threshold)
+        manager.handleRequestSync(from: peer, request: request)
+
+        try await TestHelpers.waitFor({ delegate.packets.count == 1 }, timeout: TestConstants.shortTimeout)
+        let sentPacket = try #require(delegate.packets.first)
+        #expect(sentPacket.payload == Data([0x20]))
+    }
+
+    @Test func buildGcsPayloadHonorsSinceTimestamp() throws {
+        var config = GossipSyncManager.Config()
+        config.seenCapacity = 5
+        config.fragmentCapacity = 0
+        config.fileTransferCapacity = 0
+        config.messageSyncIntervalSeconds = 0
+        config.fragmentSyncIntervalSeconds = 0
+        config.fileTransferSyncIntervalSeconds = 0
+        config.gcsTargetFpr = 0.000001
+
+        let requestSyncManager = RequestSyncManager()
+        let manager = GossipSyncManager(myPeerID: myPeerID, config: config, requestSyncManager: requestSyncManager)
+
+        let sender = try #require(Data(hexString: "1122334455667788"))
+        let threshold = UInt64(Date().timeIntervalSince1970 * 1000)
+
+        let oldMessage = BitchatPacket(
+            type: MessageType.message.rawValue,
+            senderID: sender,
+            recipientID: nil,
+            timestamp: threshold - 1,
+            payload: Data([0xA0]),
+            signature: nil,
+            ttl: 1
+        )
+
+        let freshMessage = BitchatPacket(
+            type: MessageType.message.rawValue,
+            senderID: sender,
+            recipientID: nil,
+            timestamp: threshold + 1,
+            payload: Data([0xB0]),
+            signature: nil,
+            ttl: 1
+        )
+
+        manager.onPublicPacketSeen(oldMessage)
+        manager.onPublicPacketSeen(freshMessage)
+        manager._performMaintenanceSynchronously()
+
+        let payload = manager._buildGcsPayloadSynchronously(for: .message, sinceTimestamp: threshold)
+        let request = try #require(RequestSyncPacket.decode(from: payload))
+        let sorted = GCSFilter.decodeToSortedSet(p: request.p, m: request.m, data: request.data)
+        let oldBucket = GCSFilter.bucket(for: PacketIdUtil.computeId(oldMessage), modulus: request.m)
+        let freshBucket = GCSFilter.bucket(for: PacketIdUtil.computeId(freshMessage), modulus: request.m)
+
+        #expect(request.types?.contains(.message) == true)
+        #expect(request.sinceTimestamp == threshold)
+        #expect(GCSFilter.contains(sortedValues: sorted, candidate: freshBucket))
+        #expect(GCSFilter.contains(sortedValues: sorted, candidate: oldBucket) == false)
+    }
 }

 private final class RecordingDelegate: GossipSyncManager.Delegate {
@@ -0,0 +1,118 @@
+//
+// RequestSyncPacketTests.swift
+// bitchat
+//
+// This is free and unencumbered software released into the public domain.
+// For more information, see <https://unlicense.org>
+//
+
+import Foundation
+import Testing
+import BitFoundation
+@testable import bitchat
+
+struct RequestSyncPacketTests {
+    @Test func baseFieldsRoundTrip() throws {
+        let original = RequestSyncPacket(
+            p: 7,
+            m: 12_800,
+            data: Data([1, 2, 3, 4, 5])
+        )
+
+        let decoded = try #require(RequestSyncPacket.decode(from: original.encode()))
+
+        #expect(decoded.p == 7)
+        #expect(decoded.m == 12_800)
+        #expect(decoded.data == Data([1, 2, 3, 4, 5]))
+        #expect(decoded.types == nil)
+        #expect(decoded.sinceTimestamp == nil)
+    }
+
+    @Test func upgradedFieldsRoundTripAsAndroidWantedTypes() throws {
+        let original = RequestSyncPacket(
+            p: 8,
+            m: 25_600,
+            data: Data([10, 20, 30]),
+            types: .publicMessages,
+            sinceTimestamp: 1_700_000_000_000
+        )
+
+        let encoded = original.encode()
+        let wantedTypes = try #require(tlvValue(type: 0x04, in: encoded))
+        let decoded = try #require(RequestSyncPacket.decode(from: encoded))
+
+        #expect(wantedTypes == Data([MessageType.announce.rawValue, MessageType.message.rawValue]))
+        #expect(decoded.p == 8)
+        #expect(decoded.m == 25_600)
+        #expect(decoded.data == Data([10, 20, 30]))
+        #expect(decoded.types?.contains(.announce) == true)
+        #expect(decoded.types?.contains(.message) == true)
+        #expect(decoded.sinceTimestamp == 1_700_000_000_000)
+    }
+
+    @Test func decodesLegacyPayloadWithoutUpgradeFields() throws {
+        let payload = Data([
+            0x01, 0x00, 0x01, 0x07,
+            0x02, 0x00, 0x04, 0x00, 0x00, 0x32, 0x00,
+            0x03, 0x00, 0x03, 0x01, 0x02, 0x03
+        ])
+
+        let decoded = try #require(RequestSyncPacket.decode(from: payload))
+
+        #expect(decoded.p == 7)
+        #expect(decoded.m == 12_800)
+        #expect(decoded.data == Data([1, 2, 3]))
+        #expect(decoded.types == nil)
+        #expect(decoded.sinceTimestamp == nil)
+    }
+
+    @Test func decodesAndroidWantedTypesAndMinTimestamp() throws {
+        let payload = Data([
+            0x01, 0x00, 0x01, 0x08,
+            0x02, 0x00, 0x04, 0x00, 0x00, 0x64, 0x00,
+            0x03, 0x00, 0x02, 0xAA, 0xBB,
+            0x04, 0x00, 0x02, MessageType.announce.rawValue, MessageType.message.rawValue,
+            0x05, 0x00, 0x08, 0x00, 0x00, 0x01, 0x8B, 0xCF, 0xE5, 0x68, 0x00
+        ])
+
+        let decoded = try #require(RequestSyncPacket.decode(from: payload))
+
+        #expect(decoded.p == 8)
+        #expect(decoded.m == 25_600)
+        #expect(decoded.data == Data([0xAA, 0xBB]))
+        #expect(decoded.types?.contains(.announce) == true)
+        #expect(decoded.types?.contains(.message) == true)
+        #expect(decoded.sinceTimestamp == 1_700_000_000_000)
+    }
+
+    @Test func decodesLegacyIOSPublicMessageBitfield() throws {
+        let payload = Data([
+            0x01, 0x00, 0x01, 0x08,
+            0x02, 0x00, 0x04, 0x00, 0x00, 0x64, 0x00,
+            0x03, 0x00, 0x00,
+            0x04, 0x00, 0x01, 0x03
+        ])
+
+        let decoded = try #require(RequestSyncPacket.decode(from: payload))
+
+        #expect(decoded.types?.contains(.announce) == true)
+        #expect(decoded.types?.contains(.message) == true)
+    }
+
+    private func tlvValue(type: UInt8, in data: Data) -> Data? {
+        var offset = 0
+        while offset + 3 <= data.count {
+            let currentType = data[offset]
+            offset += 1
+            let length = (Int(data[offset]) << 8) | Int(data[offset + 1])
+            offset += 2
+            guard offset + length <= data.count else { return nil }
+            let value = data.subdata(in: offset..<(offset + length))
+            offset += length
+            if currentType == type {
+                return value
+            }
+        }
+        return nil
+    }
+}