From 9dc0ba699198ceba91fd7d193ac3fd78c16a773e Mon Sep 17 00:00:00 2001 From: jack <212554440+jackjackbits@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:50:48 +0200 Subject: [PATCH] Fix CI app-suite deadlock: cooperative-pool-blocking thread-safety tests (#1339) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Exclude perf baselines from parallel CI via --skip; sample hung tests Every app-suite CI run since the perf baselines landed (#1335) has timed out at the 15-minute job limit — main has been red for five consecutive runs. The job logs show the PerformanceBaselineTests fixtures dispatched into the parallel phase despite the BITCHAT_SKIP_PERF_BASELINES env guard from #1336, followed by ~11 minutes of silence until the timeout kills swiftpm-testing. The suite passes locally in seconds with identical flags, so the hang is specific to the CI toolchain/runners — consistent with the known XCTest-measure-under-parallel-workers hang the serial step was created to avoid. Two changes: - Exclude the baselines from the parallel phase with --skip at the SPM level, which removes them from the worker processes entirely instead of relying on the env guard reaching setUpWithError. They still run (and gate) in the dedicated serial step. - Wrap the parallel run in a 10-minute watchdog that samples any still-running test processes before killing them, so if anything else ever hangs, the run fails fast with thread stacks in the log instead of a silent 15-minute timeout. Co-Authored-By: Claude Fable 5 * Arm the test-hang watchdog only for the test execution phase Codex review: swift test builds before running, so the watchdog timer included dependency resolution and compilation — a cold-cache coverage build on a slow runner could be killed before tests ever started. Build the tests in their own step (bounded by the 15-minute job timeout like any build) and run the watchdog around swift test --skip-build, tightened to 5 minutes now that it times only test execution. Co-Authored-By: Claude Fable 5 * Hammer transport thread-safety tests from the dispatch pool, not the cooperative pool The watchdog added in this PR captured the actual CI hang twice, with identical stacks both times: NostrTransportTests' 100-task groups park every Swift Concurrency cooperative thread in a blocking queue.sync (the pool has one thread per core — 3 on CI runners, 10+ on dev machines, which is why this never reproduced locally). Blocking the entire cooperative pool violates the forward-progress contract, and the runners' dispatch wedges under the resulting asyncAndWait flood — taking concurrently running tests down with it (the panic-reset test deadlocked in a serviceQueue barrier that never got scheduled). Run the same 100 concurrent hammer iterations via DispatchQueue.concurrentPerform from a single global-queue hop instead: identical thread-safety coverage, executed on dispatch worker threads where blocking is legal, zero cooperative threads parked. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: jack Co-authored-by: Claude Fable 5 --- .github/workflows/swift-tests.yml | 40 ++++++++++++++++++- .../Services/NostrTransportTests.swift | 31 +++++++++----- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/.github/workflows/swift-tests.yml b/.github/workflows/swift-tests.yml index 70279702..479ec616 100644 --- a/.github/workflows/swift-tests.yml +++ b/.github/workflows/swift-tests.yml @@ -41,14 +41,50 @@ jobs: ${{ runner.os }}-${{ matrix.name }}-${{ hashFiles(format('{0}/**/Package.resolved', matrix.path)) }} ${{ runner.os }}-${{ matrix.name }}- + - name: Build tests + # Built separately so the hang watchdog below times only test + # execution: a cold-cache coverage build on a slow runner can + # legitimately take several minutes, and is already bounded by the + # 15-minute job timeout. + run: swift build --build-tests --enable-code-coverage --package-path ${{ matrix.path }} + - name: Run Tests # Perf benchmarks are excluded here and run in their own serial step # below: measuring while parallel test processes contend for cores # produces noisy numbers, and the XCTest measure machinery has hung - # intermittently under parallel workers on loaded runners. + # intermittently under parallel workers on loaded runners. Excluded + # via --skip (not just the env guard): every app run since the + # baselines landed timed out at the 15-minute job limit with the + # baseline tests dispatched into the parallel phase. + # + # The watchdog samples any still-running test processes after 5 + # minutes (the suite passes in seconds when healthy; the build is + # done by this step) and kills the run, so a hang fails fast with + # stacks in the log instead of a silent timeout. env: BITCHAT_SKIP_PERF_BASELINES: "1" - run: swift test --parallel --quiet --enable-code-coverage --package-path ${{ matrix.path }} + run: | + swift test --skip-build --parallel --quiet --enable-code-coverage \ + --skip PerformanceBaselineTests \ + --package-path ${{ matrix.path }} & + test_pid=$! + ( + sleep 300 + if kill -0 "$test_pid" 2>/dev/null; then + echo "::group::Tests still running after 5 minutes — sampling before kill" + for pid in $(pgrep -if 'swiftpm-testing|xctest|PackageTests' || true); do + echo "--- sample of pid $pid ---" + sample "$pid" 5 2>/dev/null || true + done + echo "::endgroup::" + pkill -KILL -P "$test_pid" 2>/dev/null || true + kill -KILL "$test_pid" 2>/dev/null || true + fi + ) & + watchdog_pid=$! + wait "$test_pid" && status=0 || status=$? + kill "$watchdog_pid" 2>/dev/null || true + exit "$status" # Benchmarks run serially on an otherwise idle runner for stable # numbers; BITCHAT_PERF_LOG captures the PERF[...] lines for the gate. diff --git a/bitchatTests/Services/NostrTransportTests.swift b/bitchatTests/Services/NostrTransportTests.swift index 8cde7bf8..18bd6d59 100644 --- a/bitchatTests/Services/NostrTransportTests.swift +++ b/bitchatTests/Services/NostrTransportTests.swift @@ -310,6 +310,15 @@ struct NostrTransportTests { withExtendedLifetime(transport) {} } + // These thread-safety tests must hammer from the dispatch pool + // (concurrentPerform), NOT a task group: transport calls block in + // queue.sync, and a 100-task group runs them on the Swift Concurrency + // cooperative pool — one thread per core, just 3 on CI runners. Parking + // every cooperative thread in a blocking sync violates the forward + // progress contract and wedged dispatch on the CI runners' macOS, + // deadlocking the whole app suite into the 15-minute job timeout + // (watchdog stacks: NostrTransport.isPeerReachable syncs holding all + // pool threads). Blocking is legal on dispatch worker threads. @Test("Concurrent read receipt enqueue does not crash") @MainActor func concurrentReadReceiptEnqueue() async throws { @@ -318,9 +327,9 @@ struct NostrTransportTests { let transport = NostrTransport(keychain: keychain, idBridge: idBridge) let iterations = 100 - await withTaskGroup(of: Void.self) { group in - for i in 0..) in + DispatchQueue.global().async { + DispatchQueue.concurrentPerform(iterations: iterations) { i in let receipt = ReadReceipt( originalMessageID: UUID().uuidString, readerID: PeerID(str: String(format: "%016x", i)), @@ -329,8 +338,10 @@ struct NostrTransportTests { let peerID = PeerID(str: String(format: "%016x", i)) transport.sendReadReceipt(receipt, to: peerID) } + continuation.resume() } } + withExtendedLifetime(transport) {} } @Test("isPeerReachable is thread safe") @@ -341,18 +352,16 @@ struct NostrTransportTests { let transport = NostrTransport(keychain: keychain, idBridge: idBridge) let iterations = 100 - await withTaskGroup(of: Bool.self) { group in - for i in 0..) in + DispatchQueue.global().async { + DispatchQueue.concurrentPerform(iterations: iterations) { i in let peerID = PeerID(str: String(format: "%016x", i)) - return transport.isPeerReachable(peerID) + #expect(transport.isPeerReachable(peerID) == false) } - } - - for await result in group { - #expect(result == false) + continuation.resume() } } + withExtendedLifetime(transport) {} } @MainActor