mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-25 06:05:20 +00:00
Consolidate message deduplication into shared MessageDeduplicator
- Extend MessageDeduplicator with configurable maxAge/maxCount params - Add timestampFor() method for content key time-window checks - Add record() method to store entries with specific timestamps - Replace ChatViewModel's custom contentLRU implementation with MessageDeduplicator - Remove ~35 lines of duplicate LRU code from ChatViewModel This consolidates 2 separate deduplication implementations into one reusable, thread-safe component.
This commit is contained in:
@@ -2,38 +2,98 @@ import Foundation
|
||||
|
||||
// MARK: - Message Deduplicator (shared)
|
||||
|
||||
/// Thread-safe deduplicator with LRU eviction and time-based expiry.
|
||||
/// Used for both message ID deduplication (network layer) and content key deduplication (UI layer).
|
||||
final class MessageDeduplicator {
|
||||
private struct Entry {
|
||||
let messageID: String
|
||||
let id: String
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
private var entries: [Entry] = []
|
||||
private var head: Int = 0
|
||||
private var lookup = Set<String>()
|
||||
private var lookup: [String: Date] = [:] // id -> timestamp for O(1) lookup
|
||||
private let lock = NSLock()
|
||||
private let maxAge: TimeInterval = TransportConfig.messageDedupMaxAgeSeconds // 5 minutes
|
||||
private let maxCount = TransportConfig.messageDedupMaxCount
|
||||
private let maxAge: TimeInterval
|
||||
private let maxCount: Int
|
||||
|
||||
/// Initialize with default config from TransportConfig
|
||||
convenience init() {
|
||||
self.init(
|
||||
maxAge: TransportConfig.messageDedupMaxAgeSeconds,
|
||||
maxCount: TransportConfig.messageDedupMaxCount
|
||||
)
|
||||
}
|
||||
|
||||
/// Initialize with custom config for content deduplication
|
||||
init(maxAge: TimeInterval, maxCount: Int) {
|
||||
self.maxAge = maxAge
|
||||
self.maxCount = maxCount
|
||||
}
|
||||
|
||||
/// Check if message is duplicate and add if not
|
||||
func isDuplicate(_ messageID: String) -> Bool {
|
||||
func isDuplicate(_ id: String) -> Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
cleanupOldEntries()
|
||||
|
||||
if lookup.contains(messageID) {
|
||||
if lookup[id] != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
entries.append(Entry(messageID: messageID, timestamp: Date()))
|
||||
lookup.insert(messageID)
|
||||
let now = Date()
|
||||
entries.append(Entry(id: id, timestamp: now))
|
||||
lookup[id] = now
|
||||
trimIfNeeded()
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/// Record an ID with a specific timestamp (for content key tracking)
|
||||
func record(_ id: String, timestamp: Date) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
if lookup[id] == nil {
|
||||
entries.append(Entry(id: id, timestamp: timestamp))
|
||||
}
|
||||
lookup[id] = timestamp
|
||||
trimIfNeeded()
|
||||
}
|
||||
|
||||
/// Add an ID without checking (for announce-back tracking)
|
||||
func markProcessed(_ id: String) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
if lookup[id] == nil {
|
||||
let now = Date()
|
||||
entries.append(Entry(id: id, timestamp: now))
|
||||
lookup[id] = now
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if ID exists without adding
|
||||
func contains(_ id: String) -> Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return lookup[id] != nil
|
||||
}
|
||||
|
||||
/// Get timestamp for an ID (for content deduplication time-window checks)
|
||||
func timestampFor(_ id: String) -> Date? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return lookup[id]
|
||||
}
|
||||
|
||||
private func trimIfNeeded() {
|
||||
// Soft-cap and advance head by a chunk to avoid O(n) shifting
|
||||
if (entries.count - head) > maxCount {
|
||||
let removeCount = min(100, entries.count - head)
|
||||
for i in head..<(head + removeCount) {
|
||||
lookup.remove(entries[i].messageID)
|
||||
lookup.removeValue(forKey: entries[i].id)
|
||||
}
|
||||
head += removeCount
|
||||
// Periodically compact to reclaim memory
|
||||
@@ -42,26 +102,6 @@ final class MessageDeduplicator {
|
||||
head = 0
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/// Add an ID without checking (for announce-back tracking)
|
||||
func markProcessed(_ messageID: String) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
if !lookup.contains(messageID) {
|
||||
entries.append(Entry(messageID: messageID, timestamp: Date()))
|
||||
lookup.insert(messageID)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if ID exists without adding
|
||||
func contains(_ messageID: String) -> Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return lookup.contains(messageID)
|
||||
}
|
||||
|
||||
/// Clear all entries
|
||||
@@ -89,7 +129,7 @@ final class MessageDeduplicator {
|
||||
private func cleanupOldEntries() {
|
||||
let cutoff = Date().addingTimeInterval(-maxAge)
|
||||
while head < entries.count, entries[head].timestamp < cutoff {
|
||||
lookup.remove(entries[head].messageID)
|
||||
lookup.removeValue(forKey: entries[head].id)
|
||||
head += 1
|
||||
}
|
||||
if head > 0 && head > entries.count / 2 {
|
||||
|
||||
@@ -187,40 +187,11 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
|
||||
return String(format: "h:%016llx", h)
|
||||
}
|
||||
|
||||
// Persistent recent content map (LRU) to speed near-duplicate checks
|
||||
private var contentLRUMap: [String: Date] = [:]
|
||||
private var contentLRUOrder: [String] = []
|
||||
private var contentLRUHead = 0
|
||||
private let contentLRUCap = TransportConfig.contentLRUCap
|
||||
private func recordContentKey(_ key: String, timestamp: Date) {
|
||||
if contentLRUMap[key] == nil { contentLRUOrder.append(key) }
|
||||
contentLRUMap[key] = timestamp
|
||||
trimContentLRUIfNeeded()
|
||||
}
|
||||
|
||||
private func trimContentLRUIfNeeded() {
|
||||
let activeCount = contentLRUOrder.count - contentLRUHead
|
||||
guard activeCount > contentLRUCap else { return }
|
||||
|
||||
let overflow = activeCount - contentLRUCap
|
||||
for _ in 0..<overflow {
|
||||
guard let victim = popOldestContentKey() else { break }
|
||||
contentLRUMap.removeValue(forKey: victim)
|
||||
}
|
||||
}
|
||||
|
||||
private func popOldestContentKey() -> String? {
|
||||
guard contentLRUHead < contentLRUOrder.count else { return nil }
|
||||
let victim = contentLRUOrder[contentLRUHead]
|
||||
contentLRUHead += 1
|
||||
|
||||
// Periodically compact the backing storage to avoid unbounded growth.
|
||||
if contentLRUHead >= 32 && contentLRUHead * 2 >= contentLRUOrder.count {
|
||||
contentLRUOrder.removeFirst(contentLRUHead)
|
||||
contentLRUHead = 0
|
||||
}
|
||||
return victim
|
||||
}
|
||||
// Content deduplication using shared MessageDeduplicator
|
||||
private let contentDeduplicator = MessageDeduplicator(
|
||||
maxAge: 86400, // 24 hours - content dedup is primarily count-based
|
||||
maxCount: TransportConfig.contentLRUCap
|
||||
)
|
||||
// MARK: - Published Properties
|
||||
|
||||
@Published var messages: [BitchatMessage] = []
|
||||
@@ -1424,9 +1395,9 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
|
||||
timelineStore.append(message, to: activeChannel)
|
||||
refreshVisibleMessages(from: activeChannel)
|
||||
|
||||
// Update content LRU for near-dup detection
|
||||
// Update content deduplicator for near-dup detection
|
||||
let ckey = normalizedContentKey(message.content)
|
||||
recordContentKey(ckey, timestamp: message.timestamp)
|
||||
contentDeduplicator.record(ckey, timestamp: message.timestamp)
|
||||
|
||||
trimMessagesIfNeeded()
|
||||
|
||||
@@ -2454,7 +2425,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
|
||||
}
|
||||
|
||||
let key = normalizedContentKey(message.content)
|
||||
recordContentKey(key, timestamp: timestamp)
|
||||
contentDeduplicator.record(key, timestamp: timestamp)
|
||||
objectWillChange.send()
|
||||
return message
|
||||
}
|
||||
@@ -5184,7 +5155,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv
|
||||
refreshVisibleMessages(from: activeChannel)
|
||||
// Track the content key so relayed copies of the same system-style message are ignored
|
||||
let contentKey = normalizedContentKey(systemMessage.content)
|
||||
recordContentKey(contentKey, timestamp: systemMessage.timestamp)
|
||||
contentDeduplicator.record(contentKey, timestamp: systemMessage.timestamp)
|
||||
trimMessagesIfNeeded()
|
||||
objectWillChange.send()
|
||||
}
|
||||
@@ -6120,11 +6091,11 @@ extension ChatViewModel: PublicMessagePipelineDelegate {
|
||||
}
|
||||
|
||||
func pipeline(_ pipeline: PublicMessagePipeline, contentTimestampForKey key: String) -> Date? {
|
||||
contentLRUMap[key]
|
||||
contentDeduplicator.timestampFor(key)
|
||||
}
|
||||
|
||||
func pipeline(_ pipeline: PublicMessagePipeline, recordContentKey key: String, timestamp: Date) {
|
||||
recordContentKey(key, timestamp: timestamp)
|
||||
contentDeduplicator.record(key, timestamp: timestamp)
|
||||
}
|
||||
|
||||
func pipelineTrimMessages(_ pipeline: PublicMessagePipeline) {
|
||||
|
||||
Reference in New Issue
Block a user