diff --git a/bitchat/Utils/MessageDeduplicator.swift b/bitchat/Utils/MessageDeduplicator.swift index 4fefdfc6..01534307 100644 --- a/bitchat/Utils/MessageDeduplicator.swift +++ b/bitchat/Utils/MessageDeduplicator.swift @@ -2,38 +2,98 @@ import Foundation // MARK: - Message Deduplicator (shared) +/// Thread-safe deduplicator with LRU eviction and time-based expiry. +/// Used for both message ID deduplication (network layer) and content key deduplication (UI layer). final class MessageDeduplicator { private struct Entry { - let messageID: String + let id: String let timestamp: Date } private var entries: [Entry] = [] private var head: Int = 0 - private var lookup = Set() + private var lookup: [String: Date] = [:] // id -> timestamp for O(1) lookup private let lock = NSLock() - private let maxAge: TimeInterval = TransportConfig.messageDedupMaxAgeSeconds // 5 minutes - private let maxCount = TransportConfig.messageDedupMaxCount + private let maxAge: TimeInterval + private let maxCount: Int + + /// Initialize with default config from TransportConfig + convenience init() { + self.init( + maxAge: TransportConfig.messageDedupMaxAgeSeconds, + maxCount: TransportConfig.messageDedupMaxCount + ) + } + + /// Initialize with custom config for content deduplication + init(maxAge: TimeInterval, maxCount: Int) { + self.maxAge = maxAge + self.maxCount = maxCount + } /// Check if message is duplicate and add if not - func isDuplicate(_ messageID: String) -> Bool { + func isDuplicate(_ id: String) -> Bool { lock.lock() defer { lock.unlock() } cleanupOldEntries() - if lookup.contains(messageID) { + if lookup[id] != nil { return true } - entries.append(Entry(messageID: messageID, timestamp: Date())) - lookup.insert(messageID) + let now = Date() + entries.append(Entry(id: id, timestamp: now)) + lookup[id] = now + trimIfNeeded() + return false + } + + /// Record an ID with a specific timestamp (for content key tracking) + func record(_ id: String, timestamp: Date) { + lock.lock() + defer { lock.unlock() } + + if lookup[id] == nil { + entries.append(Entry(id: id, timestamp: timestamp)) + } + lookup[id] = timestamp + trimIfNeeded() + } + + /// Add an ID without checking (for announce-back tracking) + func markProcessed(_ id: String) { + lock.lock() + defer { lock.unlock() } + + if lookup[id] == nil { + let now = Date() + entries.append(Entry(id: id, timestamp: now)) + lookup[id] = now + } + } + + /// Check if ID exists without adding + func contains(_ id: String) -> Bool { + lock.lock() + defer { lock.unlock() } + return lookup[id] != nil + } + + /// Get timestamp for an ID (for content deduplication time-window checks) + func timestampFor(_ id: String) -> Date? { + lock.lock() + defer { lock.unlock() } + return lookup[id] + } + + private func trimIfNeeded() { // Soft-cap and advance head by a chunk to avoid O(n) shifting if (entries.count - head) > maxCount { let removeCount = min(100, entries.count - head) for i in head..<(head + removeCount) { - lookup.remove(entries[i].messageID) + lookup.removeValue(forKey: entries[i].id) } head += removeCount // Periodically compact to reclaim memory @@ -42,26 +102,6 @@ final class MessageDeduplicator { head = 0 } } - - return false - } - - /// Add an ID without checking (for announce-back tracking) - func markProcessed(_ messageID: String) { - lock.lock() - defer { lock.unlock() } - - if !lookup.contains(messageID) { - entries.append(Entry(messageID: messageID, timestamp: Date())) - lookup.insert(messageID) - } - } - - /// Check if ID exists without adding - func contains(_ messageID: String) -> Bool { - lock.lock() - defer { lock.unlock() } - return lookup.contains(messageID) } /// Clear all entries @@ -89,7 +129,7 @@ final class MessageDeduplicator { private func cleanupOldEntries() { let cutoff = Date().addingTimeInterval(-maxAge) while head < entries.count, entries[head].timestamp < cutoff { - lookup.remove(entries[head].messageID) + lookup.removeValue(forKey: entries[head].id) head += 1 } if head > 0 && head > entries.count / 2 { diff --git a/bitchat/ViewModels/ChatViewModel.swift b/bitchat/ViewModels/ChatViewModel.swift index 0e7bcd61..266562f0 100644 --- a/bitchat/ViewModels/ChatViewModel.swift +++ b/bitchat/ViewModels/ChatViewModel.swift @@ -187,40 +187,11 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv return String(format: "h:%016llx", h) } - // Persistent recent content map (LRU) to speed near-duplicate checks - private var contentLRUMap: [String: Date] = [:] - private var contentLRUOrder: [String] = [] - private var contentLRUHead = 0 - private let contentLRUCap = TransportConfig.contentLRUCap - private func recordContentKey(_ key: String, timestamp: Date) { - if contentLRUMap[key] == nil { contentLRUOrder.append(key) } - contentLRUMap[key] = timestamp - trimContentLRUIfNeeded() - } - - private func trimContentLRUIfNeeded() { - let activeCount = contentLRUOrder.count - contentLRUHead - guard activeCount > contentLRUCap else { return } - - let overflow = activeCount - contentLRUCap - for _ in 0.. String? { - guard contentLRUHead < contentLRUOrder.count else { return nil } - let victim = contentLRUOrder[contentLRUHead] - contentLRUHead += 1 - - // Periodically compact the backing storage to avoid unbounded growth. - if contentLRUHead >= 32 && contentLRUHead * 2 >= contentLRUOrder.count { - contentLRUOrder.removeFirst(contentLRUHead) - contentLRUHead = 0 - } - return victim - } + // Content deduplication using shared MessageDeduplicator + private let contentDeduplicator = MessageDeduplicator( + maxAge: 86400, // 24 hours - content dedup is primarily count-based + maxCount: TransportConfig.contentLRUCap + ) // MARK: - Published Properties @Published var messages: [BitchatMessage] = [] @@ -1424,9 +1395,9 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv timelineStore.append(message, to: activeChannel) refreshVisibleMessages(from: activeChannel) - // Update content LRU for near-dup detection + // Update content deduplicator for near-dup detection let ckey = normalizedContentKey(message.content) - recordContentKey(ckey, timestamp: message.timestamp) + contentDeduplicator.record(ckey, timestamp: message.timestamp) trimMessagesIfNeeded() @@ -2454,7 +2425,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv } let key = normalizedContentKey(message.content) - recordContentKey(key, timestamp: timestamp) + contentDeduplicator.record(key, timestamp: timestamp) objectWillChange.send() return message } @@ -5184,7 +5155,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv refreshVisibleMessages(from: activeChannel) // Track the content key so relayed copies of the same system-style message are ignored let contentKey = normalizedContentKey(systemMessage.content) - recordContentKey(contentKey, timestamp: systemMessage.timestamp) + contentDeduplicator.record(contentKey, timestamp: systemMessage.timestamp) trimMessagesIfNeeded() objectWillChange.send() } @@ -6120,11 +6091,11 @@ extension ChatViewModel: PublicMessagePipelineDelegate { } func pipeline(_ pipeline: PublicMessagePipeline, contentTimestampForKey key: String) -> Date? { - contentLRUMap[key] + contentDeduplicator.timestampFor(key) } func pipeline(_ pipeline: PublicMessagePipeline, recordContentKey key: String, timestamp: Date) { - recordContentKey(key, timestamp: timestamp) + contentDeduplicator.record(key, timestamp: timestamp) } func pipelineTrimMessages(_ pipeline: PublicMessagePipeline) {