Consolidate message deduplication into shared MessageDeduplicator

- Extend MessageDeduplicator with configurable maxAge/maxCount params
- Add timestampFor() method for content key time-window checks
- Add record() method to store entries with specific timestamps
- Replace ChatViewModel's custom contentLRU implementation with MessageDeduplicator
- Remove ~35 lines of duplicate LRU code from ChatViewModel

This consolidates 2 separate deduplication implementations into one
reusable, thread-safe component.
This commit is contained in:
Claude
2025-11-25 01:19:26 +00:00
parent 86f46c8b90
commit 0492480598
2 changed files with 81 additions and 70 deletions
+70 -30
View File
@@ -2,38 +2,98 @@ import Foundation
// MARK: - Message Deduplicator (shared)
/// Thread-safe deduplicator with LRU eviction and time-based expiry.
/// Used for both message ID deduplication (network layer) and content key deduplication (UI layer).
final class MessageDeduplicator {
private struct Entry {
let messageID: String
let id: String
let timestamp: Date
}
private var entries: [Entry] = []
private var head: Int = 0
private var lookup = Set<String>()
private var lookup: [String: Date] = [:] // id -> timestamp for O(1) lookup
private let lock = NSLock()
private let maxAge: TimeInterval = TransportConfig.messageDedupMaxAgeSeconds // 5 minutes
private let maxCount = TransportConfig.messageDedupMaxCount
private let maxAge: TimeInterval
private let maxCount: Int
/// Initialize with default config from TransportConfig
convenience init() {
self.init(
maxAge: TransportConfig.messageDedupMaxAgeSeconds,
maxCount: TransportConfig.messageDedupMaxCount
)
}
/// Initialize with custom config for content deduplication
init(maxAge: TimeInterval, maxCount: Int) {
self.maxAge = maxAge
self.maxCount = maxCount
}
/// Check if message is duplicate and add if not
func isDuplicate(_ messageID: String) -> Bool {
func isDuplicate(_ id: String) -> Bool {
lock.lock()
defer { lock.unlock() }
cleanupOldEntries()
if lookup.contains(messageID) {
if lookup[id] != nil {
return true
}
entries.append(Entry(messageID: messageID, timestamp: Date()))
lookup.insert(messageID)
let now = Date()
entries.append(Entry(id: id, timestamp: now))
lookup[id] = now
trimIfNeeded()
return false
}
/// Record an ID with a specific timestamp (for content key tracking)
func record(_ id: String, timestamp: Date) {
lock.lock()
defer { lock.unlock() }
if lookup[id] == nil {
entries.append(Entry(id: id, timestamp: timestamp))
}
lookup[id] = timestamp
trimIfNeeded()
}
/// Add an ID without checking (for announce-back tracking)
func markProcessed(_ id: String) {
lock.lock()
defer { lock.unlock() }
if lookup[id] == nil {
let now = Date()
entries.append(Entry(id: id, timestamp: now))
lookup[id] = now
}
}
/// Check if ID exists without adding
func contains(_ id: String) -> Bool {
lock.lock()
defer { lock.unlock() }
return lookup[id] != nil
}
/// Get timestamp for an ID (for content deduplication time-window checks)
func timestampFor(_ id: String) -> Date? {
lock.lock()
defer { lock.unlock() }
return lookup[id]
}
private func trimIfNeeded() {
// Soft-cap and advance head by a chunk to avoid O(n) shifting
if (entries.count - head) > maxCount {
let removeCount = min(100, entries.count - head)
for i in head..<(head + removeCount) {
lookup.remove(entries[i].messageID)
lookup.removeValue(forKey: entries[i].id)
}
head += removeCount
// Periodically compact to reclaim memory
@@ -42,26 +102,6 @@ final class MessageDeduplicator {
head = 0
}
}
return false
}
/// Add an ID without checking (for announce-back tracking)
func markProcessed(_ messageID: String) {
lock.lock()
defer { lock.unlock() }
if !lookup.contains(messageID) {
entries.append(Entry(messageID: messageID, timestamp: Date()))
lookup.insert(messageID)
}
}
/// Check if ID exists without adding
func contains(_ messageID: String) -> Bool {
lock.lock()
defer { lock.unlock() }
return lookup.contains(messageID)
}
/// Clear all entries
@@ -89,7 +129,7 @@ final class MessageDeduplicator {
private func cleanupOldEntries() {
let cutoff = Date().addingTimeInterval(-maxAge)
while head < entries.count, entries[head].timestamp < cutoff {
lookup.remove(entries[head].messageID)
lookup.removeValue(forKey: entries[head].id)
head += 1
}
if head > 0 && head > entries.count / 2 {
+11 -40
View File
@@ -187,40 +187,11 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
return String(format: "h:%016llx", h)
}
// Persistent recent content map (LRU) to speed near-duplicate checks
private var contentLRUMap: [String: Date] = [:]
private var contentLRUOrder: [String] = []
private var contentLRUHead = 0
private let contentLRUCap = TransportConfig.contentLRUCap
private func recordContentKey(_ key: String, timestamp: Date) {
if contentLRUMap[key] == nil { contentLRUOrder.append(key) }
contentLRUMap[key] = timestamp
trimContentLRUIfNeeded()
}
private func trimContentLRUIfNeeded() {
let activeCount = contentLRUOrder.count - contentLRUHead
guard activeCount > contentLRUCap else { return }
let overflow = activeCount - contentLRUCap
for _ in 0..<overflow {
guard let victim = popOldestContentKey() else { break }
contentLRUMap.removeValue(forKey: victim)
}
}
private func popOldestContentKey() -> String? {
guard contentLRUHead < contentLRUOrder.count else { return nil }
let victim = contentLRUOrder[contentLRUHead]
contentLRUHead += 1
// Periodically compact the backing storage to avoid unbounded growth.
if contentLRUHead >= 32 && contentLRUHead * 2 >= contentLRUOrder.count {
contentLRUOrder.removeFirst(contentLRUHead)
contentLRUHead = 0
}
return victim
}
// Content deduplication using shared MessageDeduplicator
private let contentDeduplicator = MessageDeduplicator(
maxAge: 86400, // 24 hours - content dedup is primarily count-based
maxCount: TransportConfig.contentLRUCap
)
// MARK: - Published Properties
@Published var messages: [BitchatMessage] = []
@@ -1424,9 +1395,9 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
timelineStore.append(message, to: activeChannel)
refreshVisibleMessages(from: activeChannel)
// Update content LRU for near-dup detection
// Update content deduplicator for near-dup detection
let ckey = normalizedContentKey(message.content)
recordContentKey(ckey, timestamp: message.timestamp)
contentDeduplicator.record(ckey, timestamp: message.timestamp)
trimMessagesIfNeeded()
@@ -2454,7 +2425,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
}
let key = normalizedContentKey(message.content)
recordContentKey(key, timestamp: timestamp)
contentDeduplicator.record(key, timestamp: timestamp)
objectWillChange.send()
return message
}
@@ -5184,7 +5155,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
refreshVisibleMessages(from: activeChannel)
// Track the content key so relayed copies of the same system-style message are ignored
let contentKey = normalizedContentKey(systemMessage.content)
recordContentKey(contentKey, timestamp: systemMessage.timestamp)
contentDeduplicator.record(contentKey, timestamp: systemMessage.timestamp)
trimMessagesIfNeeded()
objectWillChange.send()
}
@@ -6120,11 +6091,11 @@ extension ChatViewModel: PublicMessagePipelineDelegate {
}
func pipeline(_ pipeline: PublicMessagePipeline, contentTimestampForKey key: String) -> Date? {
contentLRUMap[key]
contentDeduplicator.timestampFor(key)
}
func pipeline(_ pipeline: PublicMessagePipeline, recordContentKey key: String, timestamp: Date) {
recordContentKey(key, timestamp: timestamp)
contentDeduplicator.record(key, timestamp: timestamp)
}
func pipelineTrimMessages(_ pipeline: PublicMessagePipeline) {