Make the flake class unrepeatable, not just fixed

Raising four deadlines fixed the four tests that happened to fire. The
class was still there: fourteen separate waitUntil helpers, most defaulting
to 1.0s, plus wait call sites with literal budgets. The fifth instance
would have landed the same way, on someone else's unrelated PR.

The rule, now written down in TestConstants.settleTimeout: **a wait
deadline is not a latency budget.** It exists so a genuine hang eventually
fails the suite, so size it for the worst-case scheduler, never for how
long the operation should take. Waits return as soon as their condition
holds, so a generous deadline is free in the passing case and only extends
genuine failures.

- Every wait helper now defaults to TestConstants.settleTimeout (30s), and
  the literal wait call sites below the floor were converted too — sixteen
  sites across ten files.
- TestTimingHygieneTests enforces it by scanning the test sources: wait
  defaults and wait call sites must be at least minimumSettleTimeout, and
  no test may assert an upper bound on elapsed wall-clock time (the
  assertion that started this, which cannot separate correct behaviour from
  a slow machine).
- Both rules waive per line with "test-timing-ok: <reason>", accepted on
  the line or in the comment block above it so the reason has room to be a
  sentence. One legitimate use so far: a NEGATIVE wait in NoiseCoverageTests
  asserting a promotion has *not* completed within 50ms, where a long
  deadline would only make the suite slow while still passing.

Injected production timeouts are deliberately not matched — the Noise
handshake timeouts are the behaviour under test, and short values are
correct there.

Verified the guard actually fails: a canary file with both banned shapes
was flagged with file and line, and the suite went green again once it was
removed. Full suite passes with all 18 cores saturated.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jack
2026-07-26 22:23:40 +01:00
co-authored by Claude Opus 5
parent 44fadeeaba
commit ac13a0c22f
19 changed files with 221 additions and 30 deletions
+1 -1
View File
@@ -392,7 +392,7 @@ final class NearbyNotesCounterTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
+9 -5
View File
@@ -723,9 +723,9 @@ struct NoiseCoverageTests {
// A failed startup requirement must not strand a late thread in
// the blocking test double after the test has returned.
oldSession.resumeDecrypt()
_ = decryptResult.wait(timeout: 5)
_ = decryptResult.wait(timeout: TestConstants.settleTimeout)
if let promotionResultForCleanup {
_ = promotionResultForCleanup.wait(timeout: 5)
_ = promotionResultForCleanup.wait(timeout: TestConstants.settleTimeout)
}
}
@@ -751,15 +751,19 @@ struct NoiseCoverageTests {
promotionThread.name = "NoiseCoverageTests.staleDecrypt.promote"
promotionThread.qualityOfService = .userInitiated
promotionThread.start()
try #require(promotionStarted.wait(timeout: .now() + 5) == .success)
try #require(promotionStarted.wait(timeout: .now() + TestConstants.settleTimeout) == .success)
#expect(
// test-timing-ok: a NEGATIVE wait it asserts the promotion has
// NOT completed yet, so a long deadline would only make the suite
// slow while still passing. A starved runner can only make this
// more likely to hold, never less.
promotionResult.wait(timeout: 0.05) == nil,
"Promotion must wait for the exact decrypting-session lease"
)
oldSession.resumeDecrypt()
let decrypted = try #require(decryptResult.wait(timeout: 5)).get()
_ = try #require(promotionResult.wait(timeout: 5)).get()
let decrypted = try #require(decryptResult.wait(timeout: TestConstants.settleTimeout)).get()
_ = try #require(promotionResult.wait(timeout: TestConstants.settleTimeout)).get()
#expect(decrypted.plaintext == Data("old session".utf8))
#expect(decrypted.sessionGeneration == oldGeneration)
@@ -589,7 +589,7 @@ final class GeoRelayDirectoryTests: XCTestCase {
/// a starved background task. Returning as soon as the condition holds means
/// a longer deadline only extends the genuine-failure case.
private func waitUntil(
timeout: TimeInterval = 30.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () async -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
+1 -1
View File
@@ -188,7 +188,7 @@ struct PTTBurstPlayerTests {
_ condition: () -> Bool,
sourceLocation: SourceLocation = #_sourceLocation
) async {
let deadline = ContinuousClock.now.advanced(by: .seconds(5))
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
while !condition(), ContinuousClock.now < deadline {
await Task.yield()
try? await Task.sleep(nanoseconds: 1_000_000)
@@ -15,7 +15,7 @@ final class FavoritesPersistenceServiceTests: XCTestCase {
service.addFavorite(peerNoisePublicKey: peerKey, peerNostrPublicKey: "npub1alice", peerNickname: "Alice")
wait(for: [expectation], timeout: 1.0)
wait(for: [expectation], timeout: TestConstants.settleTimeout)
XCTAssertTrue(service.isFavorite(peerKey))
XCTAssertEqual(service.getFavoriteStatus(for: peerKey)?.peerNickname, "Alice")
XCTAssertNotNil(keychain.load(key: storageKey, service: serviceKey))
@@ -227,7 +227,7 @@ final class GeohashPresenceServiceTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -355,7 +355,7 @@ final class LocationStateManagerTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -63,7 +63,7 @@ final class NetworkActivationServiceTests: XCTestCase {
context.service.start()
context.service.setUserTorEnabled(false)
wait(for: [notified], timeout: 1.0)
wait(for: [notified], timeout: TestConstants.settleTimeout)
context.notificationCenter.removeObserver(token)
XCTAssertFalse(context.service.userTorEnabled)
@@ -243,7 +243,7 @@ final class NetworkActivationServiceTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -199,7 +199,7 @@ final class NetworkReachabilityGateTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -105,11 +105,11 @@ struct NoiseEncryptionServiceTests {
try establishSessions(alice: alice, bob: bob)
let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: 5.0)
let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: TestConstants.settleTimeout)
#expect(authenticated)
let generationAuthenticated = await TestHelpers.waitUntil(
{ recorder.generationCount >= 1 },
timeout: 5.0
timeout: TestConstants.settleTimeout
)
#expect(generationAuthenticated)
#expect(alice.hasEstablishedSession(with: bobPeerID))
@@ -920,7 +920,7 @@ final class NostrRelayManagerTests: XCTestCase {
try context.sessionFactory.latestConnection(for: relayURL)?.emitEventMessage(subscriptionID: "ordered", event: event)
}
let allDelivered = await waitUntil(timeout: 5.0) {
let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) {
receivedIDs.count == events.count
}
XCTAssertTrue(allDelivered)
@@ -966,7 +966,7 @@ final class NostrRelayManagerTests: XCTestCase {
}
try context.sessionFactory.latestConnection(for: quietRelayURL)?.emitEventMessage(subscriptionID: "quiet", event: quietEvent)
let quietDelivered = await waitUntil(timeout: 5.0) { quietDeliveredAfterBusyCount >= 0 }
let quietDelivered = await waitUntil(timeout: TestConstants.settleTimeout) { quietDeliveredAfterBusyCount >= 0 }
XCTAssertTrue(quietDelivered, "relay B's event was never delivered")
// The signal: B did not have to wait for A's entire backlog. If the two
@@ -979,7 +979,7 @@ final class NostrRelayManagerTests: XCTestCase {
)
// Both relays still drain fully and in order.
let allDelivered = await waitUntil(timeout: 5.0) {
let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) {
busyDeliveredCount == busyEvents.count
}
XCTAssertTrue(allDelivered)
@@ -1867,7 +1867,7 @@ final class NostrRelayManagerTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping @MainActor () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -164,7 +164,7 @@ struct NostrTransportTests {
transport.sendPrivateMessage("hello over nostr", to: shortPeerID, recipientNickname: "Carol", messageID: "pm-1")
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
#expect(didSend)
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
let privateMessage = try decodePrivateMessage(from: result.payload)
@@ -209,7 +209,7 @@ struct NostrTransportTests {
transport.sendFavoriteNotification(to: fullPeerID, isFavorite: true)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
#expect(didSend)
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
let privateMessage = try decodePrivateMessage(from: result.payload)
@@ -250,7 +250,7 @@ struct NostrTransportTests {
transport.sendDeliveryAck(for: "ack-1", to: fullPeerID)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
#expect(didSend)
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
@@ -288,7 +288,7 @@ struct NostrTransportTests {
messageID: "geo-1"
)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
#expect(didSend)
let event = probe.sentEvents[0]
let result = try decodeEmbeddedPayload(from: event, recipient: recipient)
@@ -562,7 +562,7 @@ final class SecureIdentityStateManagerTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -320,7 +320,7 @@ struct SecureIdentityStateManagerVouchTests {
// MARK: - Helpers
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
@@ -63,7 +63,7 @@ final class RequestSyncManagerTests: XCTestCase {
}
private func waitUntil(
timeout: TimeInterval = 1.0,
timeout: TimeInterval = TestConstants.settleTimeout,
condition: @escaping () -> Bool
) async -> Bool {
let deadline = Date().addingTimeInterval(timeout)
+27 -1
View File
@@ -18,7 +18,33 @@ struct TestConstants {
/// `defaultTimeout`. `waitUntil` returns as soon as the condition holds,
/// so passing runs never pay the longer timeout.
static let longTimeout: TimeInterval = 10.0
/// **Default deadline for any "wait until this async thing settles" helper.**
///
/// Four separate tests flaked on CI during July 2026 with the same root
/// cause, and it is worth stating the rule rather than re-learning it a
/// fifth time: *a wait deadline is not a latency budget.* It exists so a
/// genuine hang eventually fails the suite. Size it for the worst-case
/// scheduler, never for how long the operation "should" take.
///
/// A CI runner executes many suites at once. Work behind `@MainActor`,
/// `Task.detached(priority: .utility)`, or a `DispatchQueue.asyncAfter` can
/// be starved for seconds one observed run took 3.75 s for a 1 s
/// operation. Deadlines sized to the operation (the old 1 s defaults) turn
/// that starvation into a red build that reads like a product bug.
///
/// This costs nothing when tests pass, because every helper returns as soon
/// as its condition holds. It only extends the genuine-failure case.
///
/// `TestTimingHygieneTests` enforces that wait helpers default to at least
/// `minimumSettleTimeout`.
static let settleTimeout: TimeInterval = 30.0
/// Floor enforced by `TestTimingHygieneTests`. Anything below this is a
/// latency assumption in disguise.
static let minimumSettleTimeout: TimeInterval = 10.0
static let testNickname1 = "Alice"
static let testNickname2 = "Bob"
static let testNickname3 = "Charlie"
@@ -0,0 +1,161 @@
import Foundation
import Testing
/// Guards the test suite against the flake class that produced four separate
/// red builds in July 2026: **treating a wait deadline as a latency budget.**
///
/// A CI runner executes many suites at once, so work behind `@MainActor`,
/// `Task.detached(priority: .utility)`, or `DispatchQueue.asyncAfter` can be
/// starved for seconds. One observed run took 3.75 s for a 1 s operation.
/// Deadlines sized to how long the operation "should" take turn that starvation
/// into a red build that reads like a product bug, and the debugging cost lands
/// on whoever opened an unrelated PR.
///
/// Two rules, both enforced below:
///
/// 1. A wait helper's default deadline must be at least
/// `TestConstants.minimumSettleTimeout`. Waits return as soon as their
/// condition holds, so a generous deadline is free in the passing case.
/// 2. No test asserts an *upper bound* on elapsed wall-clock time. Such an
/// assertion cannot distinguish the behaviour under test from a slow
/// machine, so it can only be flaky. Assert the property somewhere it is
/// computable with an injected clock, on the pure logic instead.
///
/// Both rules can be waived per line with `\(Self.waiver)` plus a reason, for
/// the rare case where the timing itself is genuinely the thing under test.
struct TestTimingHygieneTests {
/// Opt-out marker. Reviewers should expect a reason next to it.
static let waiver = "test-timing-ok:"
private static let testsRoot = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent() // TestUtilities
.deletingLastPathComponent() // bitchatTests
private struct Line {
let file: String
let number: Int
let text: String
/// True when the waiver appears on this line or in the comment block
/// immediately above it, so a reason can be written at readable length
/// rather than crammed onto the end of the code line.
let waived: Bool
}
private static func swiftLines() throws -> [Line] {
let enumerator = FileManager.default.enumerator(
at: testsRoot,
includingPropertiesForKeys: nil
)
var out: [Line] = []
while let url = enumerator?.nextObject() as? URL {
guard url.pathExtension == "swift" else { continue }
// This file necessarily contains the patterns it bans.
guard url.lastPathComponent != "TestTimingHygieneTests.swift" else { continue }
let name = url.lastPathComponent
let texts = try String(contentsOf: url, encoding: .utf8)
.components(separatedBy: .newlines)
for (index, text) in texts.enumerated() {
// Scan back over an unbroken run of comment lines.
var waived = text.contains(waiver)
var back = index - 1
while !waived, back >= 0 {
let above = texts[back].trimmingCharacters(in: .whitespaces)
guard above.hasPrefix("//") else { break }
waived = above.contains(waiver)
back -= 1
}
out.append(Line(file: name, number: index + 1, text: text, waived: waived))
}
}
return out
}
private static func isWaived(_ line: Line) -> Bool {
line.waived
}
/// Rule 1: no wait helper may default to a deadline below the floor.
@Test func waitHelpersDoNotDefaultToShortDeadlines() throws {
let lines = try Self.swiftLines()
#expect(!lines.isEmpty, "hygiene scan found no test sources — check the path")
// Two shapes, both of which have flaked here:
// a declaration default `timeout: TimeInterval = 2.5`
// a wait call site `wait(for:, timeout: 1.0)`, `waitUntil(timeout: 5.0)`
//
// Deliberately NOT matched: a bare `timeout:` label on something that is
// not a wait, such as the injected production handshake timeouts in the
// Noise tests. Those are the behaviour under test, and a short value is
// correct there.
let patterns = [
#"(?:timeout|deadline)\s*:\s*TimeInterval\s*=\s*([0-9]+(?:\.[0-9]+)?)"#,
#"(?:wait|waitUntil|waitFor|fulfillment)\s*\([^)]*\btimeout:\s*([0-9]+(?:\.[0-9]+)?)"#
].map { try? NSRegularExpression(pattern: $0) }.compactMap { $0 }
#expect(patterns.count == 2, "hygiene regexes failed to compile")
var offenders: [String] = []
for line in lines where !Self.isWaived(line) {
let range = NSRange(line.text.startIndex..., in: line.text)
for pattern in patterns {
guard let match = pattern.firstMatch(in: line.text, range: range),
let valueRange = Range(match.range(at: 1), in: line.text),
let value = TimeInterval(line.text[valueRange]),
value < TestConstants.minimumSettleTimeout else { continue }
offenders.append("\(line.file):\(line.number)\(value)s: \(line.text.trimmingCharacters(in: .whitespaces))")
break
}
}
#expect(
offenders.isEmpty,
"""
Wait deadlines below \(TestConstants.minimumSettleTimeout)s are latency \
assumptions and will flake on a loaded runner. Use \
TestConstants.settleTimeout, or add "\(Self.waiver) <reason>" if the \
timing really is what the test asserts.
\(offenders.joined(separator: "\n"))
"""
)
}
/// Rule 2: no test bounds elapsed wall-clock time from above.
///
/// This is the assertion that started it all `XCTAssertLessThan(
/// Date().timeIntervalSince(start), 1.4)` proving a debounce deadline was
/// not restarted. It cannot separate "behaved correctly" from "runner was
/// busy", so it only ever fails for the wrong reason.
@Test func testsDoNotAssertUpperBoundsOnElapsedTime() throws {
let lines = try Self.swiftLines()
let elapsedAssertion = try NSRegularExpression(
pattern: #"(?:XCTAssertLessThan|XCTAssertLessThanOrEqual)\s*\(\s*(?:Date\(\)\.timeIntervalSince|[A-Za-z_][A-Za-z0-9_]*\.timeIntervalSince|ContinuousClock)"#
)
var offenders: [String] = []
for line in lines where !Self.isWaived(line) {
let range = NSRange(line.text.startIndex..., in: line.text)
guard elapsedAssertion.firstMatch(in: line.text, range: range) != nil else { continue }
offenders.append("\(line.file):\(line.number)\(line.text.trimmingCharacters(in: .whitespaces))")
}
#expect(
offenders.isEmpty,
"""
An upper bound on elapsed wall-clock time cannot distinguish the \
behaviour under test from a slow machine. Assert the property where \
it is computable inject a clock, or test the pure logic or add \
"\(Self.waiver) <reason>".
\(offenders.joined(separator: "\n"))
"""
)
}
/// The floor must stay meaningfully above the operations being waited on,
/// and the default must satisfy the rule this file enforces.
@Test func settleTimeoutsAreSelfConsistent() {
#expect(TestConstants.settleTimeout >= TestConstants.minimumSettleTimeout)
#expect(TestConstants.minimumSettleTimeout > TestConstants.defaultTimeout)
}
}
+1 -1
View File
@@ -101,7 +101,7 @@ struct VoiceCaptureSessionTests {
_ condition: () -> Bool,
sourceLocation: SourceLocation = #_sourceLocation
) async {
let deadline = ContinuousClock.now.advanced(by: .seconds(5))
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
while !condition(), ContinuousClock.now < deadline {
await Task.yield()
try? await Task.sleep(nanoseconds: 1_000_000)
@@ -76,7 +76,7 @@ struct VoiceNotePlaybackControllerTests {
_ condition: () -> Bool,
sourceLocation: SourceLocation = #_sourceLocation
) async {
let deadline = ContinuousClock.now.advanced(by: .seconds(30))
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
while !condition(), ContinuousClock.now < deadline {
await Task.yield()
try? await Task.sleep(nanoseconds: 1_000_000)