Resurrect dead Noise vector tests; add CI perf floors; make tests hermetic

Package.swift's .process("Noise") resource claim silently excluded all
of bitchatTests/Noise/ from compilation since Oct 2025 - including a
complete official-vector runner (cacophony + snow XX transcripts,
transport messages, handshake hash, byte-identical to upstream).
Narrowing the resource to the JSON file and loading via Bundle.module
brings 51 Noise tests back to life, with a guard asserting each
vector's protocol name matches the app's.

CI gains a performance floor gate: perf-floors.json carries deliberately
generous floors (~25% of measured throughput) that catch algorithmic
regressions without flaking on runner variance; PERF lines reach the
gate via an O_APPEND side-channel file since swift test --parallel
swallows passing tests' stdout.

Tests are now hermetic: FavoritesPersistenceService uses an in-memory
keychain under test (fixes the securityd hang that blocked pipeline
benchmarks locally) and read-receipt persistence uses a wiped scratch
UserDefaults suite instead of .standard.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jack
2026-06-11 15:53:34 +02:00
co-authored by Claude Fable 5
parent 8a867a17a1
commit 22be3d6392
9 changed files with 285 additions and 12 deletions
+26 -2
View File
@@ -14,6 +14,18 @@ import BitFoundation
// MARK: - Test Vector Support
/// Official Noise test vectors (NoiseTestVectors.json) for
/// `Noise_XX_25519_ChaChaPoly_SHA256` the exact protocol this app speaks
/// (see `NoiseProtocolName` / `NoisePattern.XX`). Embedded byte-for-byte from
/// the two canonical community vector suites:
/// - cacophony: https://raw.githubusercontent.com/haskell-cryptography/cacophony/master/vectors/cacophony.txt
/// (6 messages: full XX handshake transcript + 3 transport messages, with
/// `handshake_hash`)
/// - snow: https://raw.githubusercontent.com/mcginty/snow/main/tests/vectors/snow.txt
/// (5 messages: full XX handshake transcript + 2 transport messages)
/// Plain XX has no PSKs and no pre-message keys; prologue is part of both
/// vectors and is mixed via `NoiseHandshakeState(prologue:)`. Fixed ephemerals
/// come in through the `predeterminedEphemeralKey` test seam.
struct NoiseTestVector: Codable {
let protocol_name: String
let init_prologue: String
@@ -586,9 +598,16 @@ struct NoiseProtocolTests {
@Test func noiseTestVectors() throws {
// Load test vectors from bundle
let testVectors = try loadTestVectors()
#expect(!testVectors.isEmpty, "No Noise test vectors loaded from fixture")
// Every embedded vector must target the exact protocol the app uses.
let appProtocolName = NoiseProtocolName(pattern: NoisePattern.XX.patternName).fullName
for (index, testVector) in testVectors.enumerated() {
print("Running test vector \(index + 1): \(testVector.protocol_name)")
#expect(
testVector.protocol_name == appProtocolName,
"Vector \(index + 1) targets \(testVector.protocol_name), app speaks \(appProtocolName)")
try runTestVector(testVector)
}
}
@@ -612,8 +631,13 @@ struct NoiseProtocolTests {
}
private func loadTestVectors() throws -> [NoiseTestVector] {
// Try to load from test bundle
// SwiftPM puts processed resources in the module bundle; the Xcode
// test target puts them in the test bundle itself.
#if SWIFT_PACKAGE
let testBundle = Bundle.module
#else
let testBundle = Bundle(for: MockKeychain.self)
#endif
guard let url = testBundle.url(forResource: "NoiseTestVectors", withExtension: "json")
else {
throw NSError(
@@ -38,15 +38,40 @@ final class PerformanceBaselineTests: XCTestCase {
}
/// Reports one human-readable throughput line per benchmark so CI logs
/// are readable without parsing XCTest's measure output.
/// are readable without parsing XCTest's measure output. The same line is
/// appended to the file named by `BITCHAT_PERF_LOG` (if set): under
/// `swift test --parallel` the runner swallows stdout of passing tests,
/// so the CI floor gate (scripts/check-perf-floors.sh) reads the file.
private func reportThroughput(_ name: String, samples: [TimeInterval], operations: Int, unit: String) {
guard !samples.isEmpty else { return }
let avg = samples.reduce(0, +) / Double(samples.count)
let opsPerSec = avg > 0 ? Double(operations) / avg : .infinity
print(String(
let line = String(
format: "PERF[%@]: %.0f %@/sec (avg %.3f ms per pass of %d, %d passes)",
name, opsPerSec, unit, avg * 1000, operations, samples.count
))
)
print(line)
Self.appendToPerfLog(line)
}
private static var perfLogPath: String? {
let path = ProcessInfo.processInfo.environment["BITCHAT_PERF_LOG"]
return (path?.isEmpty ?? true) ? nil : path
}
/// Appends with `O_APPEND` because `swift test --parallel` may split this
/// class across worker processes that write concurrently. The file is
/// append-only (CI workspaces start fresh); delete it between local runs
/// if you reuse a path.
private static func appendToPerfLog(_ line: String) {
guard let path = perfLogPath else { return }
let fd = open(path, O_WRONLY | O_APPEND | O_CREAT, 0o644)
guard fd >= 0 else { return }
defer { close(fd) }
let bytes = Array((line + "\n").utf8)
bytes.withUnsafeBufferPointer { buffer in
_ = write(fd, buffer.baseAddress, buffer.count)
}
}
// MARK: - 1a. Nostr inbound event handling (fresh events)
+41
View File
@@ -0,0 +1,41 @@
{
"_philosophy": [
"Floor throughputs for the PERF[...] lines printed by PerformanceBaselineTests.",
"Floors catch algorithmic regressions (O(n) -> O(n^2), accidental sync I/O,",
"quadratic re-scans), NOT tuning noise: each floor is deliberately set at",
"~25% of the throughput measured on a local dev machine (2026-06, Apple",
"Silicon), leaving ~4x headroom for CI runner variance so the gate never",
"flakes on a slow runner while still failing loudly on order-of-magnitude",
"regressions.",
"Raise floors deliberately after intentional performance improvements;",
"lower them only with a written justification in the PR. If a benchmark is",
"renamed or removed, update this file in the same change - the gate fails",
"when a floored benchmark stops reporting.",
"Checked by scripts/check-perf-floors.sh against captured swift-test output."
],
"_units": "operations per second, matching each benchmark's PERF line",
"_reference_local_numbers_2026_06": {
"nostrInbound.fresh": 2132,
"nostrInbound.duplicate": 2558907,
"bleInbound.roundTripAndDedup": 38063,
"gcs.buildAndDecode": 776,
"delivery.incrementalUpdate": 172615,
"delivery.storeUpdate": 158862,
"formatting.formatMessage": 12261,
"pipeline.privateIngest": 24848,
"pipeline.publicIngest": 13102,
"store.append": 213201
},
"floors": {
"nostrInbound.fresh": 530,
"nostrInbound.duplicate": 600000,
"bleInbound.roundTripAndDedup": 9500,
"gcs.buildAndDecode": 190,
"delivery.incrementalUpdate": 43000,
"delivery.storeUpdate": 39000,
"formatting.formatMessage": 3000,
"pipeline.privateIngest": 6000,
"pipeline.publicIngest": 3200,
"store.append": 53000
}
}