From ac13a0c22f6a7135c571eb8ccb15bd44dbce9944 Mon Sep 17 00:00:00 2001 From: jack Date: Sun, 26 Jul 2026 22:23:40 +0100 Subject: [PATCH] Make the flake class unrepeatable, not just fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raising four deadlines fixed the four tests that happened to fire. The class was still there: fourteen separate waitUntil helpers, most defaulting to 1.0s, plus wait call sites with literal budgets. The fifth instance would have landed the same way, on someone else's unrelated PR. The rule, now written down in TestConstants.settleTimeout: **a wait deadline is not a latency budget.** It exists so a genuine hang eventually fails the suite, so size it for the worst-case scheduler, never for how long the operation should take. Waits return as soon as their condition holds, so a generous deadline is free in the passing case and only extends genuine failures. - Every wait helper now defaults to TestConstants.settleTimeout (30s), and the literal wait call sites below the floor were converted too — sixteen sites across ten files. - TestTimingHygieneTests enforces it by scanning the test sources: wait defaults and wait call sites must be at least minimumSettleTimeout, and no test may assert an upper bound on elapsed wall-clock time (the assertion that started this, which cannot separate correct behaviour from a slow machine). - Both rules waive per line with "test-timing-ok: ", accepted on the line or in the comment block above it so the reason has room to be a sentence. One legitimate use so far: a NEGATIVE wait in NoiseCoverageTests asserting a promotion has *not* completed within 50ms, where a long deadline would only make the suite slow while still passing. Injected production timeouts are deliberately not matched — the Noise handshake timeouts are the behaviour under test, and short values are correct there. Verified the guard actually fails: a canary file with both banned shapes was flagged with file and line, and the suite went green again once it was removed. Full suite passes with all 18 cores saturated. Co-Authored-By: Claude Opus 5 --- bitchatTests/NearbyNotesCounterTests.swift | 2 +- bitchatTests/Noise/NoiseCoverageTests.swift | 14 +- .../Nostr/GeoRelayDirectoryTests.swift | 2 +- bitchatTests/PTTBurstPlayerTests.swift | 2 +- .../FavoritesPersistenceServiceTests.swift | 2 +- .../GeohashPresenceServiceTests.swift | 2 +- .../Services/LocationStateManagerTests.swift | 2 +- .../NetworkActivationServiceTests.swift | 4 +- .../NetworkReachabilityGateTests.swift | 2 +- .../NoiseEncryptionServiceTests.swift | 4 +- .../Services/NostrRelayManagerTests.swift | 8 +- .../Services/NostrTransportTests.swift | 8 +- .../SecureIdentityStateManagerTests.swift | 2 +- ...SecureIdentityStateManagerVouchTests.swift | 2 +- .../Sync/RequestSyncManagerTests.swift | 2 +- .../TestUtilities/TestConstants.swift | 28 ++- .../TestTimingHygieneTests.swift | 161 ++++++++++++++++++ bitchatTests/VoiceCaptureSessionTests.swift | 2 +- .../VoiceNotePlaybackControllerTests.swift | 2 +- 19 files changed, 221 insertions(+), 30 deletions(-) create mode 100644 bitchatTests/TestUtilities/TestTimingHygieneTests.swift diff --git a/bitchatTests/NearbyNotesCounterTests.swift b/bitchatTests/NearbyNotesCounterTests.swift index b548a8ae..5e328197 100644 --- a/bitchatTests/NearbyNotesCounterTests.swift +++ b/bitchatTests/NearbyNotesCounterTests.swift @@ -392,7 +392,7 @@ final class NearbyNotesCounterTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Noise/NoiseCoverageTests.swift b/bitchatTests/Noise/NoiseCoverageTests.swift index 7f3eb01a..d083bd47 100644 --- a/bitchatTests/Noise/NoiseCoverageTests.swift +++ b/bitchatTests/Noise/NoiseCoverageTests.swift @@ -723,9 +723,9 @@ struct NoiseCoverageTests { // A failed startup requirement must not strand a late thread in // the blocking test double after the test has returned. oldSession.resumeDecrypt() - _ = decryptResult.wait(timeout: 5) + _ = decryptResult.wait(timeout: TestConstants.settleTimeout) if let promotionResultForCleanup { - _ = promotionResultForCleanup.wait(timeout: 5) + _ = promotionResultForCleanup.wait(timeout: TestConstants.settleTimeout) } } @@ -751,15 +751,19 @@ struct NoiseCoverageTests { promotionThread.name = "NoiseCoverageTests.staleDecrypt.promote" promotionThread.qualityOfService = .userInitiated promotionThread.start() - try #require(promotionStarted.wait(timeout: .now() + 5) == .success) + try #require(promotionStarted.wait(timeout: .now() + TestConstants.settleTimeout) == .success) #expect( + // test-timing-ok: a NEGATIVE wait — it asserts the promotion has + // NOT completed yet, so a long deadline would only make the suite + // slow while still passing. A starved runner can only make this + // more likely to hold, never less. promotionResult.wait(timeout: 0.05) == nil, "Promotion must wait for the exact decrypting-session lease" ) oldSession.resumeDecrypt() - let decrypted = try #require(decryptResult.wait(timeout: 5)).get() - _ = try #require(promotionResult.wait(timeout: 5)).get() + let decrypted = try #require(decryptResult.wait(timeout: TestConstants.settleTimeout)).get() + _ = try #require(promotionResult.wait(timeout: TestConstants.settleTimeout)).get() #expect(decrypted.plaintext == Data("old session".utf8)) #expect(decrypted.sessionGeneration == oldGeneration) diff --git a/bitchatTests/Nostr/GeoRelayDirectoryTests.swift b/bitchatTests/Nostr/GeoRelayDirectoryTests.swift index 92ef6e4f..1a4ce385 100644 --- a/bitchatTests/Nostr/GeoRelayDirectoryTests.swift +++ b/bitchatTests/Nostr/GeoRelayDirectoryTests.swift @@ -589,7 +589,7 @@ final class GeoRelayDirectoryTests: XCTestCase { /// a starved background task. Returning as soon as the condition holds means /// a longer deadline only extends the genuine-failure case. private func waitUntil( - timeout: TimeInterval = 30.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () async -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/PTTBurstPlayerTests.swift b/bitchatTests/PTTBurstPlayerTests.swift index 00881f75..77d60a69 100644 --- a/bitchatTests/PTTBurstPlayerTests.swift +++ b/bitchatTests/PTTBurstPlayerTests.swift @@ -188,7 +188,7 @@ struct PTTBurstPlayerTests { _ condition: () -> Bool, sourceLocation: SourceLocation = #_sourceLocation ) async { - let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout)) while !condition(), ContinuousClock.now < deadline { await Task.yield() try? await Task.sleep(nanoseconds: 1_000_000) diff --git a/bitchatTests/Services/FavoritesPersistenceServiceTests.swift b/bitchatTests/Services/FavoritesPersistenceServiceTests.swift index aea80020..29eff020 100644 --- a/bitchatTests/Services/FavoritesPersistenceServiceTests.swift +++ b/bitchatTests/Services/FavoritesPersistenceServiceTests.swift @@ -15,7 +15,7 @@ final class FavoritesPersistenceServiceTests: XCTestCase { service.addFavorite(peerNoisePublicKey: peerKey, peerNostrPublicKey: "npub1alice", peerNickname: "Alice") - wait(for: [expectation], timeout: 1.0) + wait(for: [expectation], timeout: TestConstants.settleTimeout) XCTAssertTrue(service.isFavorite(peerKey)) XCTAssertEqual(service.getFavoriteStatus(for: peerKey)?.peerNickname, "Alice") XCTAssertNotNil(keychain.load(key: storageKey, service: serviceKey)) diff --git a/bitchatTests/Services/GeohashPresenceServiceTests.swift b/bitchatTests/Services/GeohashPresenceServiceTests.swift index f463c4a1..127c0052 100644 --- a/bitchatTests/Services/GeohashPresenceServiceTests.swift +++ b/bitchatTests/Services/GeohashPresenceServiceTests.swift @@ -227,7 +227,7 @@ final class GeohashPresenceServiceTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/LocationStateManagerTests.swift b/bitchatTests/Services/LocationStateManagerTests.swift index 69515f76..46114d26 100644 --- a/bitchatTests/Services/LocationStateManagerTests.swift +++ b/bitchatTests/Services/LocationStateManagerTests.swift @@ -355,7 +355,7 @@ final class LocationStateManagerTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/NetworkActivationServiceTests.swift b/bitchatTests/Services/NetworkActivationServiceTests.swift index 06d3cf33..29c5bfb1 100644 --- a/bitchatTests/Services/NetworkActivationServiceTests.swift +++ b/bitchatTests/Services/NetworkActivationServiceTests.swift @@ -63,7 +63,7 @@ final class NetworkActivationServiceTests: XCTestCase { context.service.start() context.service.setUserTorEnabled(false) - wait(for: [notified], timeout: 1.0) + wait(for: [notified], timeout: TestConstants.settleTimeout) context.notificationCenter.removeObserver(token) XCTAssertFalse(context.service.userTorEnabled) @@ -243,7 +243,7 @@ final class NetworkActivationServiceTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/NetworkReachabilityGateTests.swift b/bitchatTests/Services/NetworkReachabilityGateTests.swift index 0340b9f3..525f417a 100644 --- a/bitchatTests/Services/NetworkReachabilityGateTests.swift +++ b/bitchatTests/Services/NetworkReachabilityGateTests.swift @@ -199,7 +199,7 @@ final class NetworkReachabilityGateTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/NoiseEncryptionServiceTests.swift b/bitchatTests/Services/NoiseEncryptionServiceTests.swift index 55229501..819e8d89 100644 --- a/bitchatTests/Services/NoiseEncryptionServiceTests.swift +++ b/bitchatTests/Services/NoiseEncryptionServiceTests.swift @@ -105,11 +105,11 @@ struct NoiseEncryptionServiceTests { try establishSessions(alice: alice, bob: bob) - let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: 5.0) + let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: TestConstants.settleTimeout) #expect(authenticated) let generationAuthenticated = await TestHelpers.waitUntil( { recorder.generationCount >= 1 }, - timeout: 5.0 + timeout: TestConstants.settleTimeout ) #expect(generationAuthenticated) #expect(alice.hasEstablishedSession(with: bobPeerID)) diff --git a/bitchatTests/Services/NostrRelayManagerTests.swift b/bitchatTests/Services/NostrRelayManagerTests.swift index c50d1ba4..6ac39a65 100644 --- a/bitchatTests/Services/NostrRelayManagerTests.swift +++ b/bitchatTests/Services/NostrRelayManagerTests.swift @@ -920,7 +920,7 @@ final class NostrRelayManagerTests: XCTestCase { try context.sessionFactory.latestConnection(for: relayURL)?.emitEventMessage(subscriptionID: "ordered", event: event) } - let allDelivered = await waitUntil(timeout: 5.0) { + let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) { receivedIDs.count == events.count } XCTAssertTrue(allDelivered) @@ -966,7 +966,7 @@ final class NostrRelayManagerTests: XCTestCase { } try context.sessionFactory.latestConnection(for: quietRelayURL)?.emitEventMessage(subscriptionID: "quiet", event: quietEvent) - let quietDelivered = await waitUntil(timeout: 5.0) { quietDeliveredAfterBusyCount >= 0 } + let quietDelivered = await waitUntil(timeout: TestConstants.settleTimeout) { quietDeliveredAfterBusyCount >= 0 } XCTAssertTrue(quietDelivered, "relay B's event was never delivered") // The signal: B did not have to wait for A's entire backlog. If the two @@ -979,7 +979,7 @@ final class NostrRelayManagerTests: XCTestCase { ) // Both relays still drain fully and in order. - let allDelivered = await waitUntil(timeout: 5.0) { + let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) { busyDeliveredCount == busyEvents.count } XCTAssertTrue(allDelivered) @@ -1867,7 +1867,7 @@ final class NostrRelayManagerTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping @MainActor () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/NostrTransportTests.swift b/bitchatTests/Services/NostrTransportTests.swift index e8fd19b5..5f657000 100644 --- a/bitchatTests/Services/NostrTransportTests.swift +++ b/bitchatTests/Services/NostrTransportTests.swift @@ -164,7 +164,7 @@ struct NostrTransportTests { transport.sendPrivateMessage("hello over nostr", to: shortPeerID, recipientNickname: "Carol", messageID: "pm-1") - let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0) + let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout) #expect(didSend) let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient) let privateMessage = try decodePrivateMessage(from: result.payload) @@ -209,7 +209,7 @@ struct NostrTransportTests { transport.sendFavoriteNotification(to: fullPeerID, isFavorite: true) - let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0) + let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout) #expect(didSend) let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient) let privateMessage = try decodePrivateMessage(from: result.payload) @@ -250,7 +250,7 @@ struct NostrTransportTests { transport.sendDeliveryAck(for: "ack-1", to: fullPeerID) - let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0) + let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout) #expect(didSend) let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient) @@ -288,7 +288,7 @@ struct NostrTransportTests { messageID: "geo-1" ) - let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0) + let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout) #expect(didSend) let event = probe.sentEvents[0] let result = try decodeEmbeddedPayload(from: event, recipient: recipient) diff --git a/bitchatTests/Services/SecureIdentityStateManagerTests.swift b/bitchatTests/Services/SecureIdentityStateManagerTests.swift index c56049bb..aae65dab 100644 --- a/bitchatTests/Services/SecureIdentityStateManagerTests.swift +++ b/bitchatTests/Services/SecureIdentityStateManagerTests.swift @@ -562,7 +562,7 @@ final class SecureIdentityStateManagerTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Services/SecureIdentityStateManagerVouchTests.swift b/bitchatTests/Services/SecureIdentityStateManagerVouchTests.swift index 6ab46490..ae71c91a 100644 --- a/bitchatTests/Services/SecureIdentityStateManagerVouchTests.swift +++ b/bitchatTests/Services/SecureIdentityStateManagerVouchTests.swift @@ -320,7 +320,7 @@ struct SecureIdentityStateManagerVouchTests { // MARK: - Helpers private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/Sync/RequestSyncManagerTests.swift b/bitchatTests/Sync/RequestSyncManagerTests.swift index aada4bfc..497271e6 100644 --- a/bitchatTests/Sync/RequestSyncManagerTests.swift +++ b/bitchatTests/Sync/RequestSyncManagerTests.swift @@ -63,7 +63,7 @@ final class RequestSyncManagerTests: XCTestCase { } private func waitUntil( - timeout: TimeInterval = 1.0, + timeout: TimeInterval = TestConstants.settleTimeout, condition: @escaping () -> Bool ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) diff --git a/bitchatTests/TestUtilities/TestConstants.swift b/bitchatTests/TestUtilities/TestConstants.swift index 83f5b46b..37948f92 100644 --- a/bitchatTests/TestUtilities/TestConstants.swift +++ b/bitchatTests/TestUtilities/TestConstants.swift @@ -18,7 +18,33 @@ struct TestConstants { /// `defaultTimeout`. `waitUntil` returns as soon as the condition holds, /// so passing runs never pay the longer timeout. static let longTimeout: TimeInterval = 10.0 - + + /// **Default deadline for any "wait until this async thing settles" helper.** + /// + /// Four separate tests flaked on CI during July 2026 with the same root + /// cause, and it is worth stating the rule rather than re-learning it a + /// fifth time: *a wait deadline is not a latency budget.* It exists so a + /// genuine hang eventually fails the suite. Size it for the worst-case + /// scheduler, never for how long the operation "should" take. + /// + /// A CI runner executes many suites at once. Work behind `@MainActor`, + /// `Task.detached(priority: .utility)`, or a `DispatchQueue.asyncAfter` can + /// be starved for seconds — one observed run took 3.75 s for a 1 s + /// operation. Deadlines sized to the operation (the old 1 s defaults) turn + /// that starvation into a red build that reads like a product bug. + /// + /// This costs nothing when tests pass, because every helper returns as soon + /// as its condition holds. It only extends the genuine-failure case. + /// + /// `TestTimingHygieneTests` enforces that wait helpers default to at least + /// `minimumSettleTimeout`. + static let settleTimeout: TimeInterval = 30.0 + + /// Floor enforced by `TestTimingHygieneTests`. Anything below this is a + /// latency assumption in disguise. + static let minimumSettleTimeout: TimeInterval = 10.0 + + static let testNickname1 = "Alice" static let testNickname2 = "Bob" static let testNickname3 = "Charlie" diff --git a/bitchatTests/TestUtilities/TestTimingHygieneTests.swift b/bitchatTests/TestUtilities/TestTimingHygieneTests.swift new file mode 100644 index 00000000..94a90861 --- /dev/null +++ b/bitchatTests/TestUtilities/TestTimingHygieneTests.swift @@ -0,0 +1,161 @@ +import Foundation +import Testing + +/// Guards the test suite against the flake class that produced four separate +/// red builds in July 2026: **treating a wait deadline as a latency budget.** +/// +/// A CI runner executes many suites at once, so work behind `@MainActor`, +/// `Task.detached(priority: .utility)`, or `DispatchQueue.asyncAfter` can be +/// starved for seconds. One observed run took 3.75 s for a 1 s operation. +/// Deadlines sized to how long the operation "should" take turn that starvation +/// into a red build that reads like a product bug, and the debugging cost lands +/// on whoever opened an unrelated PR. +/// +/// Two rules, both enforced below: +/// +/// 1. A wait helper's default deadline must be at least +/// `TestConstants.minimumSettleTimeout`. Waits return as soon as their +/// condition holds, so a generous deadline is free in the passing case. +/// 2. No test asserts an *upper bound* on elapsed wall-clock time. Such an +/// assertion cannot distinguish the behaviour under test from a slow +/// machine, so it can only be flaky. Assert the property somewhere it is +/// computable — with an injected clock, on the pure logic — instead. +/// +/// Both rules can be waived per line with `\(Self.waiver)` plus a reason, for +/// the rare case where the timing itself is genuinely the thing under test. +struct TestTimingHygieneTests { + /// Opt-out marker. Reviewers should expect a reason next to it. + static let waiver = "test-timing-ok:" + + private static let testsRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // TestUtilities + .deletingLastPathComponent() // bitchatTests + + private struct Line { + let file: String + let number: Int + let text: String + /// True when the waiver appears on this line or in the comment block + /// immediately above it, so a reason can be written at readable length + /// rather than crammed onto the end of the code line. + let waived: Bool + } + + private static func swiftLines() throws -> [Line] { + let enumerator = FileManager.default.enumerator( + at: testsRoot, + includingPropertiesForKeys: nil + ) + var out: [Line] = [] + while let url = enumerator?.nextObject() as? URL { + guard url.pathExtension == "swift" else { continue } + // This file necessarily contains the patterns it bans. + guard url.lastPathComponent != "TestTimingHygieneTests.swift" else { continue } + let name = url.lastPathComponent + let texts = try String(contentsOf: url, encoding: .utf8) + .components(separatedBy: .newlines) + for (index, text) in texts.enumerated() { + // Scan back over an unbroken run of comment lines. + var waived = text.contains(waiver) + var back = index - 1 + while !waived, back >= 0 { + let above = texts[back].trimmingCharacters(in: .whitespaces) + guard above.hasPrefix("//") else { break } + waived = above.contains(waiver) + back -= 1 + } + out.append(Line(file: name, number: index + 1, text: text, waived: waived)) + } + } + return out + } + + private static func isWaived(_ line: Line) -> Bool { + line.waived + } + + /// Rule 1: no wait helper may default to a deadline below the floor. + @Test func waitHelpersDoNotDefaultToShortDeadlines() throws { + let lines = try Self.swiftLines() + #expect(!lines.isEmpty, "hygiene scan found no test sources — check the path") + + // Two shapes, both of which have flaked here: + // a declaration default — `timeout: TimeInterval = 2.5` + // a wait call site — `wait(for:…, timeout: 1.0)`, `waitUntil(timeout: 5.0)` + // + // Deliberately NOT matched: a bare `timeout:` label on something that is + // not a wait, such as the injected production handshake timeouts in the + // Noise tests. Those are the behaviour under test, and a short value is + // correct there. + let patterns = [ + #"(?:timeout|deadline)\s*:\s*TimeInterval\s*=\s*([0-9]+(?:\.[0-9]+)?)"#, + #"(?:wait|waitUntil|waitFor|fulfillment)\s*\([^)]*\btimeout:\s*([0-9]+(?:\.[0-9]+)?)"# + ].map { try? NSRegularExpression(pattern: $0) }.compactMap { $0 } + #expect(patterns.count == 2, "hygiene regexes failed to compile") + + var offenders: [String] = [] + for line in lines where !Self.isWaived(line) { + let range = NSRange(line.text.startIndex..., in: line.text) + for pattern in patterns { + guard let match = pattern.firstMatch(in: line.text, range: range), + let valueRange = Range(match.range(at: 1), in: line.text), + let value = TimeInterval(line.text[valueRange]), + value < TestConstants.minimumSettleTimeout else { continue } + offenders.append("\(line.file):\(line.number) — \(value)s: \(line.text.trimmingCharacters(in: .whitespaces))") + break + } + } + + #expect( + offenders.isEmpty, + """ + Wait deadlines below \(TestConstants.minimumSettleTimeout)s are latency \ + assumptions and will flake on a loaded runner. Use \ + TestConstants.settleTimeout, or add "\(Self.waiver) " if the \ + timing really is what the test asserts. + + \(offenders.joined(separator: "\n")) + """ + ) + } + + /// Rule 2: no test bounds elapsed wall-clock time from above. + /// + /// This is the assertion that started it all — `XCTAssertLessThan( + /// Date().timeIntervalSince(start), 1.4)` proving a debounce deadline was + /// not restarted. It cannot separate "behaved correctly" from "runner was + /// busy", so it only ever fails for the wrong reason. + @Test func testsDoNotAssertUpperBoundsOnElapsedTime() throws { + let lines = try Self.swiftLines() + + let elapsedAssertion = try NSRegularExpression( + pattern: #"(?:XCTAssertLessThan|XCTAssertLessThanOrEqual)\s*\(\s*(?:Date\(\)\.timeIntervalSince|[A-Za-z_][A-Za-z0-9_]*\.timeIntervalSince|ContinuousClock)"# + ) + + var offenders: [String] = [] + for line in lines where !Self.isWaived(line) { + let range = NSRange(line.text.startIndex..., in: line.text) + guard elapsedAssertion.firstMatch(in: line.text, range: range) != nil else { continue } + offenders.append("\(line.file):\(line.number) — \(line.text.trimmingCharacters(in: .whitespaces))") + } + + #expect( + offenders.isEmpty, + """ + An upper bound on elapsed wall-clock time cannot distinguish the \ + behaviour under test from a slow machine. Assert the property where \ + it is computable — inject a clock, or test the pure logic — or add \ + "\(Self.waiver) ". + + \(offenders.joined(separator: "\n")) + """ + ) + } + + /// The floor must stay meaningfully above the operations being waited on, + /// and the default must satisfy the rule this file enforces. + @Test func settleTimeoutsAreSelfConsistent() { + #expect(TestConstants.settleTimeout >= TestConstants.minimumSettleTimeout) + #expect(TestConstants.minimumSettleTimeout > TestConstants.defaultTimeout) + } +} diff --git a/bitchatTests/VoiceCaptureSessionTests.swift b/bitchatTests/VoiceCaptureSessionTests.swift index 344697a6..bfa0ac96 100644 --- a/bitchatTests/VoiceCaptureSessionTests.swift +++ b/bitchatTests/VoiceCaptureSessionTests.swift @@ -101,7 +101,7 @@ struct VoiceCaptureSessionTests { _ condition: () -> Bool, sourceLocation: SourceLocation = #_sourceLocation ) async { - let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout)) while !condition(), ContinuousClock.now < deadline { await Task.yield() try? await Task.sleep(nanoseconds: 1_000_000) diff --git a/bitchatTests/VoiceNotePlaybackControllerTests.swift b/bitchatTests/VoiceNotePlaybackControllerTests.swift index d6221b15..29991d0e 100644 --- a/bitchatTests/VoiceNotePlaybackControllerTests.swift +++ b/bitchatTests/VoiceNotePlaybackControllerTests.swift @@ -76,7 +76,7 @@ struct VoiceNotePlaybackControllerTests { _ condition: () -> Bool, sourceLocation: SourceLocation = #_sourceLocation ) async { - let deadline = ContinuousClock.now.advanced(by: .seconds(30)) + let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout)) while !condition(), ContinuousClock.now < deadline { await Task.yield() try? await Task.sleep(nanoseconds: 1_000_000)