mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-27 12:45:20 +00:00
Make the flake class unrepeatable, not just fixed
Raising four deadlines fixed the four tests that happened to fire. The class was still there: fourteen separate waitUntil helpers, most defaulting to 1.0s, plus wait call sites with literal budgets. The fifth instance would have landed the same way, on someone else's unrelated PR. The rule, now written down in TestConstants.settleTimeout: **a wait deadline is not a latency budget.** It exists so a genuine hang eventually fails the suite, so size it for the worst-case scheduler, never for how long the operation should take. Waits return as soon as their condition holds, so a generous deadline is free in the passing case and only extends genuine failures. - Every wait helper now defaults to TestConstants.settleTimeout (30s), and the literal wait call sites below the floor were converted too — sixteen sites across ten files. - TestTimingHygieneTests enforces it by scanning the test sources: wait defaults and wait call sites must be at least minimumSettleTimeout, and no test may assert an upper bound on elapsed wall-clock time (the assertion that started this, which cannot separate correct behaviour from a slow machine). - Both rules waive per line with "test-timing-ok: <reason>", accepted on the line or in the comment block above it so the reason has room to be a sentence. One legitimate use so far: a NEGATIVE wait in NoiseCoverageTests asserting a promotion has *not* completed within 50ms, where a long deadline would only make the suite slow while still passing. Injected production timeouts are deliberately not matched — the Noise handshake timeouts are the behaviour under test, and short values are correct there. Verified the guard actually fails: a canary file with both banned shapes was flagged with file and line, and the suite went green again once it was removed. Full suite passes with all 18 cores saturated. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -392,7 +392,7 @@ final class NearbyNotesCounterTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -723,9 +723,9 @@ struct NoiseCoverageTests {
|
|||||||
// A failed startup requirement must not strand a late thread in
|
// A failed startup requirement must not strand a late thread in
|
||||||
// the blocking test double after the test has returned.
|
// the blocking test double after the test has returned.
|
||||||
oldSession.resumeDecrypt()
|
oldSession.resumeDecrypt()
|
||||||
_ = decryptResult.wait(timeout: 5)
|
_ = decryptResult.wait(timeout: TestConstants.settleTimeout)
|
||||||
if let promotionResultForCleanup {
|
if let promotionResultForCleanup {
|
||||||
_ = promotionResultForCleanup.wait(timeout: 5)
|
_ = promotionResultForCleanup.wait(timeout: TestConstants.settleTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -751,15 +751,19 @@ struct NoiseCoverageTests {
|
|||||||
promotionThread.name = "NoiseCoverageTests.staleDecrypt.promote"
|
promotionThread.name = "NoiseCoverageTests.staleDecrypt.promote"
|
||||||
promotionThread.qualityOfService = .userInitiated
|
promotionThread.qualityOfService = .userInitiated
|
||||||
promotionThread.start()
|
promotionThread.start()
|
||||||
try #require(promotionStarted.wait(timeout: .now() + 5) == .success)
|
try #require(promotionStarted.wait(timeout: .now() + TestConstants.settleTimeout) == .success)
|
||||||
#expect(
|
#expect(
|
||||||
|
// test-timing-ok: a NEGATIVE wait — it asserts the promotion has
|
||||||
|
// NOT completed yet, so a long deadline would only make the suite
|
||||||
|
// slow while still passing. A starved runner can only make this
|
||||||
|
// more likely to hold, never less.
|
||||||
promotionResult.wait(timeout: 0.05) == nil,
|
promotionResult.wait(timeout: 0.05) == nil,
|
||||||
"Promotion must wait for the exact decrypting-session lease"
|
"Promotion must wait for the exact decrypting-session lease"
|
||||||
)
|
)
|
||||||
|
|
||||||
oldSession.resumeDecrypt()
|
oldSession.resumeDecrypt()
|
||||||
let decrypted = try #require(decryptResult.wait(timeout: 5)).get()
|
let decrypted = try #require(decryptResult.wait(timeout: TestConstants.settleTimeout)).get()
|
||||||
_ = try #require(promotionResult.wait(timeout: 5)).get()
|
_ = try #require(promotionResult.wait(timeout: TestConstants.settleTimeout)).get()
|
||||||
|
|
||||||
#expect(decrypted.plaintext == Data("old session".utf8))
|
#expect(decrypted.plaintext == Data("old session".utf8))
|
||||||
#expect(decrypted.sessionGeneration == oldGeneration)
|
#expect(decrypted.sessionGeneration == oldGeneration)
|
||||||
|
|||||||
@@ -589,7 +589,7 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
|||||||
/// a starved background task. Returning as soon as the condition holds means
|
/// a starved background task. Returning as soon as the condition holds means
|
||||||
/// a longer deadline only extends the genuine-failure case.
|
/// a longer deadline only extends the genuine-failure case.
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 30.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () async -> Bool
|
condition: @escaping @MainActor () async -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ struct PTTBurstPlayerTests {
|
|||||||
_ condition: () -> Bool,
|
_ condition: () -> Bool,
|
||||||
sourceLocation: SourceLocation = #_sourceLocation
|
sourceLocation: SourceLocation = #_sourceLocation
|
||||||
) async {
|
) async {
|
||||||
let deadline = ContinuousClock.now.advanced(by: .seconds(5))
|
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
|
||||||
while !condition(), ContinuousClock.now < deadline {
|
while !condition(), ContinuousClock.now < deadline {
|
||||||
await Task.yield()
|
await Task.yield()
|
||||||
try? await Task.sleep(nanoseconds: 1_000_000)
|
try? await Task.sleep(nanoseconds: 1_000_000)
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ final class FavoritesPersistenceServiceTests: XCTestCase {
|
|||||||
|
|
||||||
service.addFavorite(peerNoisePublicKey: peerKey, peerNostrPublicKey: "npub1alice", peerNickname: "Alice")
|
service.addFavorite(peerNoisePublicKey: peerKey, peerNostrPublicKey: "npub1alice", peerNickname: "Alice")
|
||||||
|
|
||||||
wait(for: [expectation], timeout: 1.0)
|
wait(for: [expectation], timeout: TestConstants.settleTimeout)
|
||||||
XCTAssertTrue(service.isFavorite(peerKey))
|
XCTAssertTrue(service.isFavorite(peerKey))
|
||||||
XCTAssertEqual(service.getFavoriteStatus(for: peerKey)?.peerNickname, "Alice")
|
XCTAssertEqual(service.getFavoriteStatus(for: peerKey)?.peerNickname, "Alice")
|
||||||
XCTAssertNotNil(keychain.load(key: storageKey, service: serviceKey))
|
XCTAssertNotNil(keychain.load(key: storageKey, service: serviceKey))
|
||||||
|
|||||||
@@ -227,7 +227,7 @@ final class GeohashPresenceServiceTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -355,7 +355,7 @@ final class LocationStateManagerTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ final class NetworkActivationServiceTests: XCTestCase {
|
|||||||
context.service.start()
|
context.service.start()
|
||||||
context.service.setUserTorEnabled(false)
|
context.service.setUserTorEnabled(false)
|
||||||
|
|
||||||
wait(for: [notified], timeout: 1.0)
|
wait(for: [notified], timeout: TestConstants.settleTimeout)
|
||||||
context.notificationCenter.removeObserver(token)
|
context.notificationCenter.removeObserver(token)
|
||||||
|
|
||||||
XCTAssertFalse(context.service.userTorEnabled)
|
XCTAssertFalse(context.service.userTorEnabled)
|
||||||
@@ -243,7 +243,7 @@ final class NetworkActivationServiceTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -199,7 +199,7 @@ final class NetworkReachabilityGateTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -105,11 +105,11 @@ struct NoiseEncryptionServiceTests {
|
|||||||
|
|
||||||
try establishSessions(alice: alice, bob: bob)
|
try establishSessions(alice: alice, bob: bob)
|
||||||
|
|
||||||
let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: 5.0)
|
let authenticated = await TestHelpers.waitUntil({ recorder.count >= 2 }, timeout: TestConstants.settleTimeout)
|
||||||
#expect(authenticated)
|
#expect(authenticated)
|
||||||
let generationAuthenticated = await TestHelpers.waitUntil(
|
let generationAuthenticated = await TestHelpers.waitUntil(
|
||||||
{ recorder.generationCount >= 1 },
|
{ recorder.generationCount >= 1 },
|
||||||
timeout: 5.0
|
timeout: TestConstants.settleTimeout
|
||||||
)
|
)
|
||||||
#expect(generationAuthenticated)
|
#expect(generationAuthenticated)
|
||||||
#expect(alice.hasEstablishedSession(with: bobPeerID))
|
#expect(alice.hasEstablishedSession(with: bobPeerID))
|
||||||
|
|||||||
@@ -920,7 +920,7 @@ final class NostrRelayManagerTests: XCTestCase {
|
|||||||
try context.sessionFactory.latestConnection(for: relayURL)?.emitEventMessage(subscriptionID: "ordered", event: event)
|
try context.sessionFactory.latestConnection(for: relayURL)?.emitEventMessage(subscriptionID: "ordered", event: event)
|
||||||
}
|
}
|
||||||
|
|
||||||
let allDelivered = await waitUntil(timeout: 5.0) {
|
let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) {
|
||||||
receivedIDs.count == events.count
|
receivedIDs.count == events.count
|
||||||
}
|
}
|
||||||
XCTAssertTrue(allDelivered)
|
XCTAssertTrue(allDelivered)
|
||||||
@@ -966,7 +966,7 @@ final class NostrRelayManagerTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
try context.sessionFactory.latestConnection(for: quietRelayURL)?.emitEventMessage(subscriptionID: "quiet", event: quietEvent)
|
try context.sessionFactory.latestConnection(for: quietRelayURL)?.emitEventMessage(subscriptionID: "quiet", event: quietEvent)
|
||||||
|
|
||||||
let quietDelivered = await waitUntil(timeout: 5.0) { quietDeliveredAfterBusyCount >= 0 }
|
let quietDelivered = await waitUntil(timeout: TestConstants.settleTimeout) { quietDeliveredAfterBusyCount >= 0 }
|
||||||
XCTAssertTrue(quietDelivered, "relay B's event was never delivered")
|
XCTAssertTrue(quietDelivered, "relay B's event was never delivered")
|
||||||
|
|
||||||
// The signal: B did not have to wait for A's entire backlog. If the two
|
// The signal: B did not have to wait for A's entire backlog. If the two
|
||||||
@@ -979,7 +979,7 @@ final class NostrRelayManagerTests: XCTestCase {
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Both relays still drain fully and in order.
|
// Both relays still drain fully and in order.
|
||||||
let allDelivered = await waitUntil(timeout: 5.0) {
|
let allDelivered = await waitUntil(timeout: TestConstants.settleTimeout) {
|
||||||
busyDeliveredCount == busyEvents.count
|
busyDeliveredCount == busyEvents.count
|
||||||
}
|
}
|
||||||
XCTAssertTrue(allDelivered)
|
XCTAssertTrue(allDelivered)
|
||||||
@@ -1867,7 +1867,7 @@ final class NostrRelayManagerTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping @MainActor () -> Bool
|
condition: @escaping @MainActor () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ struct NostrTransportTests {
|
|||||||
|
|
||||||
transport.sendPrivateMessage("hello over nostr", to: shortPeerID, recipientNickname: "Carol", messageID: "pm-1")
|
transport.sendPrivateMessage("hello over nostr", to: shortPeerID, recipientNickname: "Carol", messageID: "pm-1")
|
||||||
|
|
||||||
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
|
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
|
||||||
#expect(didSend)
|
#expect(didSend)
|
||||||
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
||||||
let privateMessage = try decodePrivateMessage(from: result.payload)
|
let privateMessage = try decodePrivateMessage(from: result.payload)
|
||||||
@@ -209,7 +209,7 @@ struct NostrTransportTests {
|
|||||||
|
|
||||||
transport.sendFavoriteNotification(to: fullPeerID, isFavorite: true)
|
transport.sendFavoriteNotification(to: fullPeerID, isFavorite: true)
|
||||||
|
|
||||||
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
|
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
|
||||||
#expect(didSend)
|
#expect(didSend)
|
||||||
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
||||||
let privateMessage = try decodePrivateMessage(from: result.payload)
|
let privateMessage = try decodePrivateMessage(from: result.payload)
|
||||||
@@ -250,7 +250,7 @@ struct NostrTransportTests {
|
|||||||
|
|
||||||
transport.sendDeliveryAck(for: "ack-1", to: fullPeerID)
|
transport.sendDeliveryAck(for: "ack-1", to: fullPeerID)
|
||||||
|
|
||||||
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
|
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
|
||||||
#expect(didSend)
|
#expect(didSend)
|
||||||
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
let result = try decodeEmbeddedPayload(from: probe.sentEvents[0], recipient: recipient)
|
||||||
|
|
||||||
@@ -288,7 +288,7 @@ struct NostrTransportTests {
|
|||||||
messageID: "geo-1"
|
messageID: "geo-1"
|
||||||
)
|
)
|
||||||
|
|
||||||
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: 5.0)
|
let didSend = await TestHelpers.waitUntil({ probe.sentEvents.count == 1 }, timeout: TestConstants.settleTimeout)
|
||||||
#expect(didSend)
|
#expect(didSend)
|
||||||
let event = probe.sentEvents[0]
|
let event = probe.sentEvents[0]
|
||||||
let result = try decodeEmbeddedPayload(from: event, recipient: recipient)
|
let result = try decodeEmbeddedPayload(from: event, recipient: recipient)
|
||||||
|
|||||||
@@ -562,7 +562,7 @@ final class SecureIdentityStateManagerTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping () -> Bool
|
condition: @escaping () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -320,7 +320,7 @@ struct SecureIdentityStateManagerVouchTests {
|
|||||||
// MARK: - Helpers
|
// MARK: - Helpers
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping () -> Bool
|
condition: @escaping () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ final class RequestSyncManagerTests: XCTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func waitUntil(
|
private func waitUntil(
|
||||||
timeout: TimeInterval = 1.0,
|
timeout: TimeInterval = TestConstants.settleTimeout,
|
||||||
condition: @escaping () -> Bool
|
condition: @escaping () -> Bool
|
||||||
) async -> Bool {
|
) async -> Bool {
|
||||||
let deadline = Date().addingTimeInterval(timeout)
|
let deadline = Date().addingTimeInterval(timeout)
|
||||||
|
|||||||
@@ -18,7 +18,33 @@ struct TestConstants {
|
|||||||
/// `defaultTimeout`. `waitUntil` returns as soon as the condition holds,
|
/// `defaultTimeout`. `waitUntil` returns as soon as the condition holds,
|
||||||
/// so passing runs never pay the longer timeout.
|
/// so passing runs never pay the longer timeout.
|
||||||
static let longTimeout: TimeInterval = 10.0
|
static let longTimeout: TimeInterval = 10.0
|
||||||
|
|
||||||
|
/// **Default deadline for any "wait until this async thing settles" helper.**
|
||||||
|
///
|
||||||
|
/// Four separate tests flaked on CI during July 2026 with the same root
|
||||||
|
/// cause, and it is worth stating the rule rather than re-learning it a
|
||||||
|
/// fifth time: *a wait deadline is not a latency budget.* It exists so a
|
||||||
|
/// genuine hang eventually fails the suite. Size it for the worst-case
|
||||||
|
/// scheduler, never for how long the operation "should" take.
|
||||||
|
///
|
||||||
|
/// A CI runner executes many suites at once. Work behind `@MainActor`,
|
||||||
|
/// `Task.detached(priority: .utility)`, or a `DispatchQueue.asyncAfter` can
|
||||||
|
/// be starved for seconds — one observed run took 3.75 s for a 1 s
|
||||||
|
/// operation. Deadlines sized to the operation (the old 1 s defaults) turn
|
||||||
|
/// that starvation into a red build that reads like a product bug.
|
||||||
|
///
|
||||||
|
/// This costs nothing when tests pass, because every helper returns as soon
|
||||||
|
/// as its condition holds. It only extends the genuine-failure case.
|
||||||
|
///
|
||||||
|
/// `TestTimingHygieneTests` enforces that wait helpers default to at least
|
||||||
|
/// `minimumSettleTimeout`.
|
||||||
|
static let settleTimeout: TimeInterval = 30.0
|
||||||
|
|
||||||
|
/// Floor enforced by `TestTimingHygieneTests`. Anything below this is a
|
||||||
|
/// latency assumption in disguise.
|
||||||
|
static let minimumSettleTimeout: TimeInterval = 10.0
|
||||||
|
|
||||||
|
|
||||||
static let testNickname1 = "Alice"
|
static let testNickname1 = "Alice"
|
||||||
static let testNickname2 = "Bob"
|
static let testNickname2 = "Bob"
|
||||||
static let testNickname3 = "Charlie"
|
static let testNickname3 = "Charlie"
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
import Foundation
|
||||||
|
import Testing
|
||||||
|
|
||||||
|
/// Guards the test suite against the flake class that produced four separate
|
||||||
|
/// red builds in July 2026: **treating a wait deadline as a latency budget.**
|
||||||
|
///
|
||||||
|
/// A CI runner executes many suites at once, so work behind `@MainActor`,
|
||||||
|
/// `Task.detached(priority: .utility)`, or `DispatchQueue.asyncAfter` can be
|
||||||
|
/// starved for seconds. One observed run took 3.75 s for a 1 s operation.
|
||||||
|
/// Deadlines sized to how long the operation "should" take turn that starvation
|
||||||
|
/// into a red build that reads like a product bug, and the debugging cost lands
|
||||||
|
/// on whoever opened an unrelated PR.
|
||||||
|
///
|
||||||
|
/// Two rules, both enforced below:
|
||||||
|
///
|
||||||
|
/// 1. A wait helper's default deadline must be at least
|
||||||
|
/// `TestConstants.minimumSettleTimeout`. Waits return as soon as their
|
||||||
|
/// condition holds, so a generous deadline is free in the passing case.
|
||||||
|
/// 2. No test asserts an *upper bound* on elapsed wall-clock time. Such an
|
||||||
|
/// assertion cannot distinguish the behaviour under test from a slow
|
||||||
|
/// machine, so it can only be flaky. Assert the property somewhere it is
|
||||||
|
/// computable — with an injected clock, on the pure logic — instead.
|
||||||
|
///
|
||||||
|
/// Both rules can be waived per line with `\(Self.waiver)` plus a reason, for
|
||||||
|
/// the rare case where the timing itself is genuinely the thing under test.
|
||||||
|
struct TestTimingHygieneTests {
|
||||||
|
/// Opt-out marker. Reviewers should expect a reason next to it.
|
||||||
|
static let waiver = "test-timing-ok:"
|
||||||
|
|
||||||
|
private static let testsRoot = URL(fileURLWithPath: #filePath)
|
||||||
|
.deletingLastPathComponent() // TestUtilities
|
||||||
|
.deletingLastPathComponent() // bitchatTests
|
||||||
|
|
||||||
|
private struct Line {
|
||||||
|
let file: String
|
||||||
|
let number: Int
|
||||||
|
let text: String
|
||||||
|
/// True when the waiver appears on this line or in the comment block
|
||||||
|
/// immediately above it, so a reason can be written at readable length
|
||||||
|
/// rather than crammed onto the end of the code line.
|
||||||
|
let waived: Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func swiftLines() throws -> [Line] {
|
||||||
|
let enumerator = FileManager.default.enumerator(
|
||||||
|
at: testsRoot,
|
||||||
|
includingPropertiesForKeys: nil
|
||||||
|
)
|
||||||
|
var out: [Line] = []
|
||||||
|
while let url = enumerator?.nextObject() as? URL {
|
||||||
|
guard url.pathExtension == "swift" else { continue }
|
||||||
|
// This file necessarily contains the patterns it bans.
|
||||||
|
guard url.lastPathComponent != "TestTimingHygieneTests.swift" else { continue }
|
||||||
|
let name = url.lastPathComponent
|
||||||
|
let texts = try String(contentsOf: url, encoding: .utf8)
|
||||||
|
.components(separatedBy: .newlines)
|
||||||
|
for (index, text) in texts.enumerated() {
|
||||||
|
// Scan back over an unbroken run of comment lines.
|
||||||
|
var waived = text.contains(waiver)
|
||||||
|
var back = index - 1
|
||||||
|
while !waived, back >= 0 {
|
||||||
|
let above = texts[back].trimmingCharacters(in: .whitespaces)
|
||||||
|
guard above.hasPrefix("//") else { break }
|
||||||
|
waived = above.contains(waiver)
|
||||||
|
back -= 1
|
||||||
|
}
|
||||||
|
out.append(Line(file: name, number: index + 1, text: text, waived: waived))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func isWaived(_ line: Line) -> Bool {
|
||||||
|
line.waived
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rule 1: no wait helper may default to a deadline below the floor.
|
||||||
|
@Test func waitHelpersDoNotDefaultToShortDeadlines() throws {
|
||||||
|
let lines = try Self.swiftLines()
|
||||||
|
#expect(!lines.isEmpty, "hygiene scan found no test sources — check the path")
|
||||||
|
|
||||||
|
// Two shapes, both of which have flaked here:
|
||||||
|
// a declaration default — `timeout: TimeInterval = 2.5`
|
||||||
|
// a wait call site — `wait(for:…, timeout: 1.0)`, `waitUntil(timeout: 5.0)`
|
||||||
|
//
|
||||||
|
// Deliberately NOT matched: a bare `timeout:` label on something that is
|
||||||
|
// not a wait, such as the injected production handshake timeouts in the
|
||||||
|
// Noise tests. Those are the behaviour under test, and a short value is
|
||||||
|
// correct there.
|
||||||
|
let patterns = [
|
||||||
|
#"(?:timeout|deadline)\s*:\s*TimeInterval\s*=\s*([0-9]+(?:\.[0-9]+)?)"#,
|
||||||
|
#"(?:wait|waitUntil|waitFor|fulfillment)\s*\([^)]*\btimeout:\s*([0-9]+(?:\.[0-9]+)?)"#
|
||||||
|
].map { try? NSRegularExpression(pattern: $0) }.compactMap { $0 }
|
||||||
|
#expect(patterns.count == 2, "hygiene regexes failed to compile")
|
||||||
|
|
||||||
|
var offenders: [String] = []
|
||||||
|
for line in lines where !Self.isWaived(line) {
|
||||||
|
let range = NSRange(line.text.startIndex..., in: line.text)
|
||||||
|
for pattern in patterns {
|
||||||
|
guard let match = pattern.firstMatch(in: line.text, range: range),
|
||||||
|
let valueRange = Range(match.range(at: 1), in: line.text),
|
||||||
|
let value = TimeInterval(line.text[valueRange]),
|
||||||
|
value < TestConstants.minimumSettleTimeout else { continue }
|
||||||
|
offenders.append("\(line.file):\(line.number) — \(value)s: \(line.text.trimmingCharacters(in: .whitespaces))")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#expect(
|
||||||
|
offenders.isEmpty,
|
||||||
|
"""
|
||||||
|
Wait deadlines below \(TestConstants.minimumSettleTimeout)s are latency \
|
||||||
|
assumptions and will flake on a loaded runner. Use \
|
||||||
|
TestConstants.settleTimeout, or add "\(Self.waiver) <reason>" if the \
|
||||||
|
timing really is what the test asserts.
|
||||||
|
|
||||||
|
\(offenders.joined(separator: "\n"))
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rule 2: no test bounds elapsed wall-clock time from above.
|
||||||
|
///
|
||||||
|
/// This is the assertion that started it all — `XCTAssertLessThan(
|
||||||
|
/// Date().timeIntervalSince(start), 1.4)` proving a debounce deadline was
|
||||||
|
/// not restarted. It cannot separate "behaved correctly" from "runner was
|
||||||
|
/// busy", so it only ever fails for the wrong reason.
|
||||||
|
@Test func testsDoNotAssertUpperBoundsOnElapsedTime() throws {
|
||||||
|
let lines = try Self.swiftLines()
|
||||||
|
|
||||||
|
let elapsedAssertion = try NSRegularExpression(
|
||||||
|
pattern: #"(?:XCTAssertLessThan|XCTAssertLessThanOrEqual)\s*\(\s*(?:Date\(\)\.timeIntervalSince|[A-Za-z_][A-Za-z0-9_]*\.timeIntervalSince|ContinuousClock)"#
|
||||||
|
)
|
||||||
|
|
||||||
|
var offenders: [String] = []
|
||||||
|
for line in lines where !Self.isWaived(line) {
|
||||||
|
let range = NSRange(line.text.startIndex..., in: line.text)
|
||||||
|
guard elapsedAssertion.firstMatch(in: line.text, range: range) != nil else { continue }
|
||||||
|
offenders.append("\(line.file):\(line.number) — \(line.text.trimmingCharacters(in: .whitespaces))")
|
||||||
|
}
|
||||||
|
|
||||||
|
#expect(
|
||||||
|
offenders.isEmpty,
|
||||||
|
"""
|
||||||
|
An upper bound on elapsed wall-clock time cannot distinguish the \
|
||||||
|
behaviour under test from a slow machine. Assert the property where \
|
||||||
|
it is computable — inject a clock, or test the pure logic — or add \
|
||||||
|
"\(Self.waiver) <reason>".
|
||||||
|
|
||||||
|
\(offenders.joined(separator: "\n"))
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The floor must stay meaningfully above the operations being waited on,
|
||||||
|
/// and the default must satisfy the rule this file enforces.
|
||||||
|
@Test func settleTimeoutsAreSelfConsistent() {
|
||||||
|
#expect(TestConstants.settleTimeout >= TestConstants.minimumSettleTimeout)
|
||||||
|
#expect(TestConstants.minimumSettleTimeout > TestConstants.defaultTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -101,7 +101,7 @@ struct VoiceCaptureSessionTests {
|
|||||||
_ condition: () -> Bool,
|
_ condition: () -> Bool,
|
||||||
sourceLocation: SourceLocation = #_sourceLocation
|
sourceLocation: SourceLocation = #_sourceLocation
|
||||||
) async {
|
) async {
|
||||||
let deadline = ContinuousClock.now.advanced(by: .seconds(5))
|
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
|
||||||
while !condition(), ContinuousClock.now < deadline {
|
while !condition(), ContinuousClock.now < deadline {
|
||||||
await Task.yield()
|
await Task.yield()
|
||||||
try? await Task.sleep(nanoseconds: 1_000_000)
|
try? await Task.sleep(nanoseconds: 1_000_000)
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ struct VoiceNotePlaybackControllerTests {
|
|||||||
_ condition: () -> Bool,
|
_ condition: () -> Bool,
|
||||||
sourceLocation: SourceLocation = #_sourceLocation
|
sourceLocation: SourceLocation = #_sourceLocation
|
||||||
) async {
|
) async {
|
||||||
let deadline = ContinuousClock.now.advanced(by: .seconds(30))
|
let deadline = ContinuousClock.now.advanced(by: .seconds(TestConstants.settleTimeout))
|
||||||
while !condition(), ContinuousClock.now < deadline {
|
while !condition(), ContinuousClock.now < deadline {
|
||||||
await Task.yield()
|
await Task.yield()
|
||||||
try? await Task.sleep(nanoseconds: 1_000_000)
|
try? await Task.sleep(nanoseconds: 1_000_000)
|
||||||
|
|||||||
Reference in New Issue
Block a user