diff --git a/bitchat/Services/MessageDeduplicationService.swift b/bitchat/Services/MessageDeduplicationService.swift new file mode 100644 index 00000000..88f249ed --- /dev/null +++ b/bitchat/Services/MessageDeduplicationService.swift @@ -0,0 +1,274 @@ +// +// MessageDeduplicationService.swift +// bitchat +// +// Handles message deduplication using LRU caches. +// This is free and unencumbered software released into the public domain. +// + +import Foundation + +// MARK: - LRU Deduplication Cache + +/// Generic LRU (Least Recently Used) cache for deduplication. +/// Uses an efficient O(1) lookup with periodic compaction. +final class LRUDeduplicationCache { + private var map: [String: Value] = [:] + private var order: [String] = [] + private var head: Int = 0 + private let capacity: Int + + /// Creates a new LRU cache with the specified capacity. + /// - Parameter capacity: Maximum number of entries before eviction + init(capacity: Int) { + precondition(capacity > 0, "LRU cache capacity must be positive") + self.capacity = capacity + } + + /// Number of active entries in the cache + var count: Int { + order.count - head + } + + /// Checks if a key exists in the cache + func contains(_ key: String) -> Bool { + map[key] != nil + } + + /// Gets the value for a key, or nil if not present + func value(for key: String) -> Value? { + map[key] + } + + /// Records a key-value pair, updating if exists or inserting if new + func record(_ key: String, value: Value) { + if map[key] == nil { + order.append(key) + } + map[key] = value + trimIfNeeded() + } + + /// Removes a specific key from the cache + func remove(_ key: String) { + map.removeValue(forKey: key) + // Note: key remains in order array but will be skipped during eviction + } + + /// Clears all entries from the cache + func clear() { + map.removeAll() + order.removeAll() + head = 0 + } + + // MARK: - Private + + private func trimIfNeeded() { + let activeCount = order.count - head + guard activeCount > capacity else { return } + + let overflow = activeCount - capacity + for _ in 0.. String? { + // Skip keys that were already removed from map + while head < order.count { + let key = order[head] + head += 1 + + // Periodically compact the backing storage + if head >= 32 && head * 2 >= order.count { + order.removeFirst(head) + head = 0 + } + + // Only return if key is still in map + if map[key] != nil { + return key + } + } + return nil + } +} + +// MARK: - Content Normalizer + +/// Normalizes message content for near-duplicate detection. +enum ContentNormalizer { + + /// Regex to simplify HTTP URLs by stripping query strings and fragments + private static let simplifyHTTPURL: NSRegularExpression = { + try! NSRegularExpression( + pattern: "https?://[^\\s?#]+(?:[?#][^\\s]*)?", + options: [.caseInsensitive] + ) + }() + + /// Normalizes content for deduplication comparison. + /// - Parameters: + /// - content: The raw message content + /// - prefixLength: Maximum characters to consider (default from TransportConfig) + /// - Returns: A hash-based key for comparison + static func normalizedKey( + _ content: String, + prefixLength: Int = TransportConfig.contentKeyPrefixLength + ) -> String { + // Lowercase for case-insensitive comparison + let lowered = content.lowercased() + let ns = lowered as NSString + let range = NSRange(location: 0, length: ns.length) + + // Simplify URLs by stripping query/fragment + var simplified = "" + var last = 0 + for match in simplifyHTTPURL.matches(in: lowered, options: [], range: range) { + if match.range.location > last { + simplified += ns.substring(with: NSRange(location: last, length: match.range.location - last)) + } + let url = ns.substring(with: match.range) + if let queryIndex = url.firstIndex(where: { $0 == "?" || $0 == "#" }) { + simplified += String(url[.. + + /// Cache for Nostr event ID deduplication + private let nostrEventCache: LRUDeduplicationCache + + /// Cache for Nostr ACK deduplication (messageId:ackType:senderPubkey format) + private let nostrAckCache: LRUDeduplicationCache + + /// Creates a new deduplication service with specified capacities. + /// - Parameters: + /// - contentCapacity: Max entries for content cache + /// - nostrEventCapacity: Max entries for Nostr event cache + init( + contentCapacity: Int = TransportConfig.contentLRUCap, + nostrEventCapacity: Int = TransportConfig.uiProcessedNostrEventsCap + ) { + self.contentCache = LRUDeduplicationCache(capacity: contentCapacity) + self.nostrEventCache = LRUDeduplicationCache(capacity: nostrEventCapacity) + self.nostrAckCache = LRUDeduplicationCache(capacity: nostrEventCapacity) + } + + // MARK: - Content Deduplication + + /// Records content with its timestamp for near-duplicate detection. + /// - Parameters: + /// - content: The message content + /// - timestamp: When the content was received + func recordContent(_ content: String, timestamp: Date) { + let key = ContentNormalizer.normalizedKey(content) + contentCache.record(key, value: timestamp) + } + + /// Records a pre-normalized content key with its timestamp. + /// - Parameters: + /// - key: The normalized content key + /// - timestamp: When the content was received + func recordContentKey(_ key: String, timestamp: Date) { + contentCache.record(key, value: timestamp) + } + + /// Gets the timestamp for previously seen content. + /// - Parameter content: The message content + /// - Returns: The timestamp when first seen, or nil if not seen + func contentTimestamp(for content: String) -> Date? { + let key = ContentNormalizer.normalizedKey(content) + return contentCache.value(for: key) + } + + /// Gets the timestamp for a pre-normalized content key. + /// - Parameter key: The normalized content key + /// - Returns: The timestamp when first seen, or nil if not seen + func contentTimestamp(forKey key: String) -> Date? { + contentCache.value(for: key) + } + + /// Normalizes content to a deduplication key. + /// - Parameter content: The raw content + /// - Returns: A normalized hash key + func normalizedContentKey(_ content: String) -> String { + ContentNormalizer.normalizedKey(content) + } + + // MARK: - Nostr Event Deduplication + + /// Checks if a Nostr event has already been processed. + /// - Parameter eventId: The event ID + /// - Returns: true if already processed + func hasProcessedNostrEvent(_ eventId: String) -> Bool { + nostrEventCache.contains(eventId) + } + + /// Records a Nostr event as processed. + /// - Parameter eventId: The event ID + func recordNostrEvent(_ eventId: String) { + nostrEventCache.record(eventId, value: true) + } + + // MARK: - Nostr ACK Deduplication + + /// Checks if a Nostr ACK has already been processed. + /// - Parameter ackKey: The ACK key in format "messageId:ackType:senderPubkey" + /// - Returns: true if already processed + func hasProcessedNostrAck(_ ackKey: String) -> Bool { + nostrAckCache.contains(ackKey) + } + + /// Records a Nostr ACK as processed. + /// - Parameter ackKey: The ACK key in format "messageId:ackType:senderPubkey" + func recordNostrAck(_ ackKey: String) { + nostrAckCache.record(ackKey, value: true) + } + + /// Creates an ACK key from components. + static func ackKey(messageId: String, ackType: String, senderPubkey: String) -> String { + "\(messageId):\(ackType):\(senderPubkey)" + } + + // MARK: - Clear + + /// Clears all caches + func clearAll() { + contentCache.clear() + nostrEventCache.clear() + nostrAckCache.clear() + } + + /// Clears only the Nostr caches (events and ACKs) + func clearNostrCaches() { + nostrEventCache.clear() + nostrAckCache.clear() + } +} diff --git a/bitchat/ViewModels/ChatViewModel.swift b/bitchat/ViewModels/ChatViewModel.swift index 266562f0..76b2a3ca 100644 --- a/bitchat/ViewModels/ChatViewModel.swift +++ b/bitchat/ViewModels/ChatViewModel.swift @@ -119,9 +119,6 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv static let quickCashuPresence: NSRegularExpression = { try! NSRegularExpression(pattern: "\\bcashu[AB][A-Za-z0-9._-]{40,}\\b", options: []) }() - static let simplifyHTTPURL: NSRegularExpression = { - try! NSRegularExpression(pattern: "https?://[^\\s?#]+(?:[?#][^\\s]*)?", options: [.caseInsensitive]) - }() } private typealias GeoOutgoingContext = (channel: GeohashChannel, event: NostrEvent, identity: NostrIdentity, teleported: Bool) @@ -159,39 +156,6 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv return "name:" + message.sender.lowercased() } - private func normalizedContentKey(_ content: String) -> String { - // Lowercase, simplify URLs (strip query/fragment), collapse whitespace, bound length - let lowered = content.lowercased() - let ns = lowered as NSString - let range = NSRange(location: 0, length: ns.length) - var simplified = "" - var last = 0 - for m in Regexes.simplifyHTTPURL.matches(in: lowered, options: [], range: range) { - if m.range.location > last { - simplified += ns.substring(with: NSRange(location: last, length: m.range.location - last)) - } - let url = ns.substring(with: m.range) - if let q = url.firstIndex(where: { $0 == "?" || $0 == "#" }) { - simplified += String(url[..() // Simple deduplication - private var processedNostrEventOrder: [String] = [] - private var processedNostrEventHead = 0 - private let maxProcessedNostrEvents = TransportConfig.uiProcessedNostrEventsCap private let userDefaults = UserDefaults.standard private let keychain: KeychainManagerProtocol private let nicknameKey = "bitchat.nickname" @@ -444,9 +404,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv // Throttle verification response toasts per peer to avoid spam private var lastVerifyToastAt: [String: Date] = [:] - - // Track processed Nostr ACKs to avoid duplicate processing - private var processedNostrAcks: Set = [] // "messageId:ackType:senderPubkey" format + // Track which GeoDM messages we've already sent a delivery ACK for (by messageID) private var sentGeoDeliveryAcks: Set = [] @@ -492,7 +450,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv // Allow UnifiedPeerService to route favorite notifications via mesh/Nostr self.unifiedPeerService.messageRouter = self.messageRouter self.autocompleteService = AutocompleteService() - + self.deduplicationService = MessageDeduplicationService() + // Wire up dependencies self.commandProcessor.chatViewModel = self self.participantTracker.configure(context: self) @@ -882,12 +841,12 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv private func subscribeNostrEvent(_ event: NostrEvent) { guard event.kind == NostrProtocol.EventKind.ephemeralEvent.rawValue, - !processedNostrEvents.contains(event.id) + !deduplicationService.hasProcessedNostrEvent(event.id) else { return } - processedNostrEvents.insert(event.id) + deduplicationService.recordNostrEvent(event.id) if let gh = currentGeohash, let myGeoIdentity = try? idBridge.deriveIdentity(forGeohash: gh), @@ -956,8 +915,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv } private func subscribeGiftWrap(_ giftWrap: NostrEvent, id: NostrIdentity) { - guard !processedNostrEvents.contains(giftWrap.id) else { return } - recordProcessedEvent(giftWrap.id) + guard !deduplicationService.hasProcessedNostrEvent(giftWrap.id) else { return } + deduplicationService.recordNostrEvent(giftWrap.id) guard let (content, senderPubkey, rumorTs) = try? NostrProtocol.decryptPrivateMessage(giftWrap: giftWrap, recipientIdentity: id), content.hasPrefix("bitchat1:"), @@ -975,7 +934,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv switch noisePayload.type { case .privateMessage: - handlePrivateMessage(payload: noisePayload, senderPubkey: senderPubkey, convKey: convKey, id: id, messageTimestamp: messageTimestamp) + handlePrivateMessage(noisePayload, senderPubkey: senderPubkey, convKey: convKey, id: id, messageTimestamp: messageTimestamp) case .delivered: handleDelivered(noisePayload, senderPubkey: senderPubkey, convKey: convKey) case .readReceipt: @@ -985,71 +944,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv break } } - - private func handlePrivateMessage( - payload: NoisePayload, - senderPubkey: String, - convKey: PeerID, - id: NostrIdentity, - messageTimestamp: Date - ) { - guard let pm = PrivateMessagePacket.decode(from: payload.data) else { return } - let messageId = pm.messageID - - sendDeliveryAckIfNeeded(to: messageId, senderPubKey: senderPubkey, from: id) - - // Respect geohash blocks - if identityManager.isNostrBlocked(pubkeyHexLowercased: senderPubkey) { - return - } - - // Dedup storage - if privateChats[convKey]?.contains(where: { $0.id == messageId }) == true { return } - for (_, arr) in privateChats { if arr.contains(where: { $0.id == messageId }) { return } } - let senderName = displayNameForNostrPubkey(senderPubkey) - - let msg = BitchatMessage( - id: messageId, - sender: senderName, - content: pm.content, - timestamp: messageTimestamp, - isRelay: false, - isPrivate: true, - recipientNickname: nickname, - senderPeerID: convKey, - deliveryStatus: .delivered(to: nickname, at: Date()) - ) - - if privateChats[convKey] == nil { - privateChats[convKey] = [] - } - privateChats[convKey]?.append(msg) - - // pared back: omit view-state log - let isViewing = selectedPrivateChatPeer == convKey - let wasReadBefore = sentReadReceipts.contains(messageId) - let isRecentMessage = Date().timeIntervalSince(messageTimestamp) < 30 - let shouldMarkUnread = !wasReadBefore && !isViewing && isRecentMessage - if shouldMarkUnread { - unreadPrivateMessages.insert(convKey) - } - - if isViewing { - // pared back: omit pre-send READ log - sendReadReceiptIfNeeded(to: messageId, senderPubKey: senderPubkey, from: id) - } else { - // Notify for truly unread and recent messages when not viewing - if shouldMarkUnread { - NotificationService.shared.sendPrivateMessageNotification( - from: senderName, - message: pm.content, - peerID: convKey - ) - } - } - objectWillChange.send() - } - + private func sendDeliveryAckIfNeeded(to messageId: String, senderPubKey: String, from id: NostrIdentity) { guard !sentGeoDeliveryAcks.contains(messageId) else { return } let nt = NostrTransport(keychain: keychain, idBridge: idBridge) @@ -1395,9 +1290,9 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv timelineStore.append(message, to: activeChannel) refreshVisibleMessages(from: activeChannel) - // Update content deduplicator for near-dup detection - let ckey = normalizedContentKey(message.content) - contentDeduplicator.record(ckey, timestamp: message.timestamp) + // Update content LRU for near-dup detection + let ckey = deduplicationService.normalizedContentKey(message.content) + deduplicationService.recordContentKey(ckey, timestamp: message.timestamp) trimMessagesIfNeeded() @@ -1471,7 +1366,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv SecureLogger.info("GeoTeleport: mark self teleported key=\(key.prefix(8))… total=\(teleportedGeo.count)", category: .session) } - recordProcessedEvent(event.id) + deduplicationService.recordNostrEvent(event.id) } @MainActor @@ -1481,8 +1376,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv activeChannel = channel publicMessagePipeline.updateActiveChannel(channel) // Reset deduplication set and optionally hydrate timeline for mesh - processedNostrEvents.removeAll() - processedNostrEventOrder.removeAll() + deduplicationService.clearNostrCaches() switch channel { case .mesh: refreshVisibleMessages(from: .mesh) @@ -1553,8 +1447,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv guard event.kind == NostrProtocol.EventKind.ephemeralEvent.rawValue else { return } // Deduplicate - if processedNostrEvents.contains(event.id) { return } - recordProcessedEvent(event.id) + if deduplicationService.hasProcessedNostrEvent(event.id) { return } + deduplicationService.recordNostrEvent(event.id) // Log incoming tags for diagnostics let tagSummary = event.tags.map { "[" + $0.joined(separator: ",") + "]" }.joined(separator: ",") @@ -1656,10 +1550,10 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv } private func handleGiftWrap(_ giftWrap: NostrEvent, id: NostrIdentity) { - if processedNostrEvents.contains(giftWrap.id) { + if deduplicationService.hasProcessedNostrEvent(giftWrap.id) { return } - recordProcessedEvent(giftWrap.id) + deduplicationService.recordNostrEvent(giftWrap.id) // Decrypt with per-geohash identity guard let (content, senderPubkey, rumorTs) = try? NostrProtocol.decryptPrivateMessage(giftWrap: giftWrap, recipientIdentity: id) else { @@ -1707,9 +1601,14 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv let messageId = pm.messageID SecureLogger.info("GeoDM: recv PM <- sender=\(senderPubkey.prefix(8))… mid=\(messageId.prefix(8))…", category: .session) - + sendDeliveryAckIfNeeded(to: messageId, senderPubKey: senderPubkey, from: id) - + + // Respect geohash blocks + if identityManager.isNostrBlocked(pubkeyHexLowercased: senderPubkey) { + return + } + // Duplicate check if privateChats[convKey]?.contains(where: { $0.id == messageId }) == true { return } for (_, arr) in privateChats { @@ -2043,36 +1942,6 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv return "anon#\(suffix)" } - // Dedup helper with small memory cap - private func recordProcessedEvent(_ id: String) { - processedNostrEvents.insert(id) - processedNostrEventOrder.append(id) - trimProcessedNostrEventsIfNeeded() - } - - private func trimProcessedNostrEventsIfNeeded() { - let activeCount = processedNostrEventOrder.count - processedNostrEventHead - guard activeCount > maxProcessedNostrEvents else { return } - - let overflow = activeCount - maxProcessedNostrEvents - for _ in 0.. String? { - guard processedNostrEventHead < processedNostrEventOrder.count else { return nil } - let value = processedNostrEventOrder[processedNostrEventHead] - processedNostrEventHead += 1 - - if processedNostrEventHead >= 32 && processedNostrEventHead * 2 >= processedNostrEventOrder.count { - processedNostrEventOrder.removeFirst(processedNostrEventHead) - processedNostrEventHead = 0 - } - return value - } - /// Sends an encrypted private message to a specific peer. /// - Parameters: /// - content: The message content to encrypt and send @@ -2424,8 +2293,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv trimMessagesIfNeeded() } - let key = normalizedContentKey(message.content) - contentDeduplicator.record(key, timestamp: timestamp) + let key = deduplicationService.normalizedContentKey(message.content) + deduplicationService.recordContentKey(key, timestamp: timestamp) objectWillChange.send() return message } @@ -3421,8 +3290,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv // Clear read receipt tracking sentReadReceipts.removeAll() - processedNostrAcks.removeAll() - + deduplicationService.clearAll() + // Clear all caches invalidateEncryptionCache() @@ -5154,8 +5023,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv timelineStore.append(systemMessage, to: activeChannel) refreshVisibleMessages(from: activeChannel) // Track the content key so relayed copies of the same system-style message are ignored - let contentKey = normalizedContentKey(systemMessage.content) - contentDeduplicator.record(contentKey, timestamp: systemMessage.timestamp) + let contentKey = deduplicationService.normalizedContentKey(systemMessage.content) + deduplicationService.recordContentKey(contentKey, timestamp: systemMessage.timestamp) trimMessagesIfNeeded() objectWillChange.send() } @@ -5233,8 +5102,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv @MainActor private func handleNostrMessage(_ giftWrap: NostrEvent) { // Simple deduplication - if processedNostrEvents.contains(giftWrap.id) { return } - processedNostrEvents.insert(giftWrap.id) + if deduplicationService.hasProcessedNostrEvent(giftWrap.id) { return } + deduplicationService.recordNostrEvent(giftWrap.id) // Client-side filtering: ignore messages older than 24 hours // Add 15 minutes buffer since gift wrap timestamps are randomized ±15 minutes @@ -5971,7 +5840,7 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, CommandContextProv let shouldRateLimit = finalMessage.sender != "system" || finalMessage.senderPeerID != nil if shouldRateLimit { let senderKey = normalizedSenderKey(for: finalMessage) - let contentKey = normalizedContentKey(finalMessage.content) + let contentKey = deduplicationService.normalizedContentKey(finalMessage.content) if !publicRateLimiter.allow(senderKey: senderKey, contentKey: contentKey) { return } } @@ -6087,15 +5956,15 @@ extension ChatViewModel: PublicMessagePipelineDelegate { } func pipeline(_ pipeline: PublicMessagePipeline, normalizeContent content: String) -> String { - normalizedContentKey(content) + deduplicationService.normalizedContentKey(content) } func pipeline(_ pipeline: PublicMessagePipeline, contentTimestampForKey key: String) -> Date? { - contentDeduplicator.timestampFor(key) + deduplicationService.contentTimestamp(forKey: key) } func pipeline(_ pipeline: PublicMessagePipeline, recordContentKey key: String, timestamp: Date) { - contentDeduplicator.record(key, timestamp: timestamp) + deduplicationService.recordContentKey(key, timestamp: timestamp) } func pipelineTrimMessages(_ pipeline: PublicMessagePipeline) { diff --git a/bitchatTests/MessageDeduplicationServiceTests.swift b/bitchatTests/MessageDeduplicationServiceTests.swift new file mode 100644 index 00000000..8f265afb --- /dev/null +++ b/bitchatTests/MessageDeduplicationServiceTests.swift @@ -0,0 +1,470 @@ +// +// MessageDeduplicationServiceTests.swift +// bitchatTests +// +// Tests for MessageDeduplicationService, LRUDeduplicationCache, and ContentNormalizer. +// This is free and unencumbered software released into the public domain. +// + +import Testing +import Foundation +@testable import bitchat + +// MARK: - LRU Deduplication Cache Tests + +struct LRUDeduplicationCacheTests { + + // MARK: - Basic Operations + + @Test func emptyCache_containsReturnsFalse() { + let cache = LRUDeduplicationCache(capacity: 10) + #expect(!cache.contains("key")) + #expect(cache.value(for: "key") == nil) + #expect(cache.count == 0) + } + + @Test func record_addsEntry() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("key1", value: 42) + + #expect(cache.contains("key1")) + #expect(cache.value(for: "key1") == 42) + #expect(cache.count == 1) + } + + @Test func record_updatesExistingEntry() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("key1", value: 42) + cache.record("key1", value: 100) + + #expect(cache.value(for: "key1") == 100) + #expect(cache.count == 1) // Should not increase count + } + + @Test func record_multipleEntries() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("a", value: "alpha") + cache.record("b", value: "beta") + cache.record("c", value: "gamma") + + #expect(cache.count == 3) + #expect(cache.value(for: "a") == "alpha") + #expect(cache.value(for: "b") == "beta") + #expect(cache.value(for: "c") == "gamma") + } + + @Test func remove_removesEntry() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("key1", value: 42) + cache.record("key2", value: 100) + + cache.remove("key1") + + #expect(!cache.contains("key1")) + #expect(cache.contains("key2")) + } + + @Test func clear_removesAllEntries() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("a", value: 1) + cache.record("b", value: 2) + cache.record("c", value: 3) + + cache.clear() + + #expect(cache.count == 0) + #expect(!cache.contains("a")) + #expect(!cache.contains("b")) + #expect(!cache.contains("c")) + } + + // MARK: - Eviction Tests + + @Test func eviction_removesOldestWhenOverCapacity() { + let cache = LRUDeduplicationCache(capacity: 3) + cache.record("a", value: 1) + cache.record("b", value: 2) + cache.record("c", value: 3) + cache.record("d", value: 4) // Should evict "a" + + #expect(cache.count == 3) + #expect(!cache.contains("a")) // Evicted + #expect(cache.contains("b")) + #expect(cache.contains("c")) + #expect(cache.contains("d")) + } + + @Test func eviction_maintainsCapacity() { + let cache = LRUDeduplicationCache(capacity: 2) + + for i in 0..<100 { + cache.record("key\(i)", value: i) + } + + #expect(cache.count == 2) + // Most recent entries should be present + #expect(cache.contains("key99")) + #expect(cache.contains("key98")) + // Older entries should be evicted + #expect(!cache.contains("key0")) + #expect(!cache.contains("key50")) + } + + @Test func eviction_capacityOfOne() { + let cache = LRUDeduplicationCache(capacity: 1) + cache.record("a", value: 1) + cache.record("b", value: 2) + + #expect(cache.count == 1) + #expect(!cache.contains("a")) + #expect(cache.contains("b")) + } + + @Test func eviction_skipsRemovedKeys() { + let cache = LRUDeduplicationCache(capacity: 3) + cache.record("a", value: 1) + cache.record("b", value: 2) + cache.record("c", value: 3) + + // Remove "a" manually + cache.remove("a") + + // Add new entry - should evict "b" (next oldest still in map) + cache.record("d", value: 4) + + // Cache should have b, c, d (a was removed) + // Actually after eviction it should have c, d and maybe b depending on implementation + #expect(!cache.contains("a")) + #expect(cache.count <= 3) + } + + // MARK: - Edge Cases + + @Test func emptyKey_works() { + let cache = LRUDeduplicationCache(capacity: 10) + cache.record("", value: 42) + + #expect(cache.contains("")) + #expect(cache.value(for: "") == 42) + } + + @Test func largeCapacity_works() { + let cache = LRUDeduplicationCache(capacity: 10000) + + for i in 0..<5000 { + cache.record("key\(i)", value: i) + } + + #expect(cache.count == 5000) + #expect(cache.contains("key0")) + #expect(cache.contains("key4999")) + } +} + +// MARK: - Content Normalizer Tests + +struct ContentNormalizerTests { + + @Test func normalizedKey_basicContent() { + let key1 = ContentNormalizer.normalizedKey("Hello World") + let key2 = ContentNormalizer.normalizedKey("Hello World") + #expect(key1 == key2) + } + + @Test func normalizedKey_caseInsensitive() { + let key1 = ContentNormalizer.normalizedKey("Hello World") + let key2 = ContentNormalizer.normalizedKey("hello world") + let key3 = ContentNormalizer.normalizedKey("HELLO WORLD") + #expect(key1 == key2) + #expect(key2 == key3) + } + + @Test func normalizedKey_whitespaceCollapsed() { + let key1 = ContentNormalizer.normalizedKey("Hello World") + let key2 = ContentNormalizer.normalizedKey("Hello World") + let key3 = ContentNormalizer.normalizedKey("Hello\t\nWorld") + #expect(key1 == key2) + #expect(key2 == key3) + } + + @Test func normalizedKey_trimmed() { + let key1 = ContentNormalizer.normalizedKey("Hello") + let key2 = ContentNormalizer.normalizedKey(" Hello ") + let key3 = ContentNormalizer.normalizedKey("\nHello\n") + #expect(key1 == key2) + #expect(key2 == key3) + } + + @Test func normalizedKey_urlQueryStripped() { + let key1 = ContentNormalizer.normalizedKey("Check https://example.com/page") + let key2 = ContentNormalizer.normalizedKey("Check https://example.com/page?query=value") + let key3 = ContentNormalizer.normalizedKey("Check https://example.com/page#anchor") + #expect(key1 == key2) + #expect(key2 == key3) + } + + @Test func normalizedKey_httpAndHttpsDistinct() { + // URL scheme is preserved + let key1 = ContentNormalizer.normalizedKey("http://example.com/page") + let key2 = ContentNormalizer.normalizedKey("https://example.com/page") + #expect(key1 != key2) + } + + @Test func normalizedKey_differentContent() { + let key1 = ContentNormalizer.normalizedKey("Hello") + let key2 = ContentNormalizer.normalizedKey("Goodbye") + #expect(key1 != key2) + } + + @Test func normalizedKey_returnsHashFormat() { + let key = ContentNormalizer.normalizedKey("Test content") + #expect(key.hasPrefix("h:")) + #expect(key.count == 18) // "h:" + 16 hex chars + } + + @Test func normalizedKey_emptyContent() { + let key = ContentNormalizer.normalizedKey("") + #expect(key.hasPrefix("h:")) + } + + @Test func normalizedKey_longContentTruncated() { + let longContent = String(repeating: "a", count: 10000) + let key1 = ContentNormalizer.normalizedKey(longContent) + let key2 = ContentNormalizer.normalizedKey(longContent + "extra") + + // Both should be the same since content is truncated before hashing + #expect(key1 == key2) + } + + @Test func normalizedKey_prefixLengthRespected() { + let content = "Short" + let key1 = ContentNormalizer.normalizedKey(content, prefixLength: 3) + let key2 = ContentNormalizer.normalizedKey(content, prefixLength: 100) + + // Different prefix lengths may produce different keys + // "sho" vs "short" + #expect(key1 != key2) + } + + @Test func normalizedKey_urlsInMiddleOfContent() { + let content1 = "Check out https://example.com/path?query=1 for more info" + let content2 = "Check out https://example.com/path for more info" + let key1 = ContentNormalizer.normalizedKey(content1) + let key2 = ContentNormalizer.normalizedKey(content2) + #expect(key1 == key2) + } + + @Test func normalizedKey_multipleUrls() { + let content1 = "Links: https://a.com?x=1 and http://b.com#y" + let content2 = "Links: https://a.com and http://b.com" + let key1 = ContentNormalizer.normalizedKey(content1) + let key2 = ContentNormalizer.normalizedKey(content2) + #expect(key1 == key2) + } +} + +// MARK: - Message Deduplication Service Tests + +struct MessageDeduplicationServiceTests { + + // MARK: - Content Deduplication + + @Test func recordContent_storesTimestamp() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + + service.recordContent("Hello World", timestamp: now) + + let retrieved = service.contentTimestamp(for: "Hello World") + #expect(retrieved == now) + } + + @Test func recordContent_updatesTimestamp() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let early = Date(timeIntervalSince1970: 1000) + let late = Date(timeIntervalSince1970: 2000) + + service.recordContent("Hello World", timestamp: early) + service.recordContent("Hello World", timestamp: late) + + let retrieved = service.contentTimestamp(for: "Hello World") + #expect(retrieved == late) + } + + @Test func contentTimestamp_nilForUnseen() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + + let timestamp = service.contentTimestamp(for: "Never seen") + #expect(timestamp == nil) + } + + @Test func recordContentKey_directKeyAccess() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + let key = service.normalizedContentKey("Test") + + service.recordContentKey(key, timestamp: now) + + #expect(service.contentTimestamp(forKey: key) == now) + } + + @Test func normalizedContentKey_consistentWithNormalizer() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let content = "Hello World" + + let serviceKey = service.normalizedContentKey(content) + let normalizerKey = ContentNormalizer.normalizedKey(content) + + #expect(serviceKey == normalizerKey) + } + + // MARK: - Nostr Event Deduplication + + @Test func recordNostrEvent_marksAsProcessed() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + + #expect(!service.hasProcessedNostrEvent("event123")) + + service.recordNostrEvent("event123") + + #expect(service.hasProcessedNostrEvent("event123")) + } + + @Test func hasProcessedNostrEvent_falseForUnseen() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + + #expect(!service.hasProcessedNostrEvent("never-seen")) + } + + @Test func nostrEvent_multipleEvents() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + + service.recordNostrEvent("event1") + service.recordNostrEvent("event2") + service.recordNostrEvent("event3") + + #expect(service.hasProcessedNostrEvent("event1")) + #expect(service.hasProcessedNostrEvent("event2")) + #expect(service.hasProcessedNostrEvent("event3")) + #expect(!service.hasProcessedNostrEvent("event4")) + } + + // MARK: - Nostr ACK Deduplication + + @Test func recordNostrAck_marksAsProcessed() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let ackKey = MessageDeduplicationService.ackKey( + messageId: "msg123", + ackType: "delivered", + senderPubkey: "pubkey456" + ) + + #expect(!service.hasProcessedNostrAck(ackKey)) + + service.recordNostrAck(ackKey) + + #expect(service.hasProcessedNostrAck(ackKey)) + } + + @Test func ackKey_format() { + let key = MessageDeduplicationService.ackKey( + messageId: "msg", + ackType: "read", + senderPubkey: "pub" + ) + #expect(key == "msg:read:pub") + } + + @Test func ackKey_differentComponents() { + let key1 = MessageDeduplicationService.ackKey(messageId: "a", ackType: "delivered", senderPubkey: "x") + let key2 = MessageDeduplicationService.ackKey(messageId: "a", ackType: "read", senderPubkey: "x") + let key3 = MessageDeduplicationService.ackKey(messageId: "b", ackType: "delivered", senderPubkey: "x") + + #expect(key1 != key2) // Different ackType + #expect(key1 != key3) // Different messageId + } + + // MARK: - Clear Operations + + @Test func clearAll_clearsEverything() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + + service.recordContent("Hello", timestamp: now) + service.recordNostrEvent("event1") + service.recordNostrAck("ack1") + + service.clearAll() + + #expect(service.contentTimestamp(for: "Hello") == nil) + #expect(!service.hasProcessedNostrEvent("event1")) + #expect(!service.hasProcessedNostrAck("ack1")) + } + + @Test func clearNostrCaches_preservesContent() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + + service.recordContent("Hello", timestamp: now) + service.recordNostrEvent("event1") + service.recordNostrAck("ack1") + + service.clearNostrCaches() + + #expect(service.contentTimestamp(for: "Hello") == now) // Preserved + #expect(!service.hasProcessedNostrEvent("event1")) // Cleared + #expect(!service.hasProcessedNostrAck("ack1")) // Cleared + } + + // MARK: - Capacity Tests + + @Test func contentCache_respectsCapacity() { + let service = MessageDeduplicationService(contentCapacity: 3, nostrEventCapacity: 100) + + service.recordContent("a", timestamp: Date()) + service.recordContent("b", timestamp: Date()) + service.recordContent("c", timestamp: Date()) + service.recordContent("d", timestamp: Date()) + + // "a" should have been evicted + #expect(service.contentTimestamp(for: "a") == nil) + #expect(service.contentTimestamp(for: "d") != nil) + } + + @Test func nostrEventCache_respectsCapacity() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 3) + + service.recordNostrEvent("e1") + service.recordNostrEvent("e2") + service.recordNostrEvent("e3") + service.recordNostrEvent("e4") + + // "e1" should have been evicted + #expect(!service.hasProcessedNostrEvent("e1")) + #expect(service.hasProcessedNostrEvent("e4")) + } + + // MARK: - Integration Tests + + @Test func realWorldDeduplication_similarMessages() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + + // Record original message + service.recordContent("Check out https://example.com/page?ref=abc", timestamp: now) + + // Same URL with different query params should match + let timestamp = service.contentTimestamp(for: "Check out https://example.com/page?ref=xyz") + #expect(timestamp == now) + } + + @Test func realWorldDeduplication_caseVariations() { + let service = MessageDeduplicationService(contentCapacity: 100, nostrEventCapacity: 100) + let now = Date() + + service.recordContent("HELLO WORLD", timestamp: now) + + #expect(service.contentTimestamp(for: "hello world") == now) + #expect(service.contentTimestamp(for: "Hello World") == now) + } +}