diff --git a/.github/workflows/fetch_georelays.yml b/.github/workflows/fetch_georelays.yml index 6aaf6693..ad5f0bb4 100644 --- a/.github/workflows/fetch_georelays.yml +++ b/.github/workflows/fetch_georelays.yml @@ -1,42 +1,228 @@ -name: Fetch GeoRelays Data +name: Propose GeoRelay Data Update on: schedule: - - cron: '0 6 * * 0' + - cron: "0 6 * * 0" workflow_dispatch: +# Default to read-only. The publishing job receives only the scopes required +# to push its branch and publish either a PR or a tracking issue. permissions: - contents: write - pull-requests: write + contents: read + +concurrency: + group: georelay-data-update + cancel-in-progress: false + +env: + SOURCE_REPOSITORY: https://github.com/permissionlesstech/georelays.git + UPDATE_BRANCH: automation/georelay-data + TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request jobs: - update-relay-data: + propose-relay-data: + name: Validate and propose relay data runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + issues: write steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout reviewed base + # Pinned actions/checkout v5 so a mutable action tag cannot change the + # code that receives this job's write-capable token. + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd with: - token: ${{ secrets.GITHUB_TOKEN }} + ref: main fetch-depth: 0 + # Do not expose the write token to fetch/validation subprocesses. + persist-credentials: false - - name: Fetch GeoRelays + - name: Test GeoRelay validator run: | - wget -q https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv - mv nostr_relays.csv ./relays/online_relays_gps.csv + set -euo pipefail + python3 -m unittest discover -s scripts/tests -p "test_*.py" -v - - name: Check for changes - id: git-check + - name: Fetch candidate over pinned HTTPS policy + id: upstream run: | - git diff --exit-code || echo "changes=true" >> $GITHUB_OUTPUT - - - name: Commit and push changes - if: steps.git-check.outputs.changes == 'true' + set -euo pipefail + source_commit=$(git ls-remote --refs "$SOURCE_REPOSITORY" refs/heads/main | awk 'NR == 1 { print $1 }') + if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::Could not resolve an immutable upstream commit" + exit 1 + fi + source_url="https://raw.githubusercontent.com/permissionlesstech/georelays/$source_commit/nostr_relays.csv" + effective_url=$(curl --fail --show-error --silent --location --proto "=https" --proto-redir "=https" --tlsv1.2 --max-time 60 --retry 3 --retry-all-errors --output "$RUNNER_TEMP/georelays-candidate.csv" --write-out "%{url_effective}" "$source_url") + if [[ "$effective_url" != "$source_url" ]]; then + echo "::error::Unexpected GeoRelay redirect target: $effective_url" + exit 1 + fi + echo "source_commit=$source_commit" >> "$GITHUB_OUTPUT" + echo "source_url=$source_url" >> "$GITHUB_OUTPUT" + + - name: Validate candidate against reviewed baseline + id: validation run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add relays/online_relays_gps.csv - git commit -m "Automated update of relay data - $(date -u)" - git push + set -euo pipefail + python3 scripts/validate_georelays.py --input "$RUNNER_TEMP/georelays-candidate.csv" --baseline relays/online_relays_gps.csv --output relays/online_relays_gps.csv --github-output "$GITHUB_OUTPUT" + + - name: Check for a reviewed-file change + id: changes + run: | + set -euo pipefail + if git diff --quiet -- relays/online_relays_gps.csv; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Upstream GeoRelay data already matches main." >> "$GITHUB_STEP_SUMMARY" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + git diff --stat -- relays/online_relays_gps.csv + fi + + - name: Push automation branch and publish review request + if: steps.changes.outputs.changed == 'true' env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ github.token }} + SOURCE_COMMIT: ${{ steps.upstream.outputs.source_commit }} + SOURCE_URL: ${{ steps.upstream.outputs.source_url }} + DATA_ROWS: ${{ steps.validation.outputs.data_rows }} + UNIQUE_RELAYS: ${{ steps.validation.outputs.unique_relays }} + DATA_SHA256: ${{ steps.validation.outputs.sha256 }} + run: | + set -euo pipefail + + # Scope credential exposure to this final publishing step. + gh auth setup-git + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + git switch -C "$UPDATE_BRANCH" + git add -- relays/online_relays_gps.csv + git diff --cached --quiet && { + echo "::error::Expected a staged GeoRelay data change" + exit 1 + } + git commit -m "Update reviewed georelay directory" -m "Upstream-commit: $SOURCE_COMMIT" + + remote_ref="refs/remotes/origin/$UPDATE_BRANCH" + if git fetch --no-tags origin "+refs/heads/$UPDATE_BRANCH:$remote_ref" 2>/dev/null; then + remote_sha=$(git rev-parse "$remote_ref") + git push --force-with-lease="refs/heads/$UPDATE_BRANCH:$remote_sha" origin "HEAD:refs/heads/$UPDATE_BRANCH" + else + git push origin "HEAD:refs/heads/$UPDATE_BRANCH" + fi + + body_file="$RUNNER_TEMP/georelay-pr-body.md" + { + echo "## Automated GeoRelay data proposal" + echo + echo "- Source: $SOURCE_URL" + echo "- Upstream commit: $SOURCE_COMMIT" + echo "- Data rows: $DATA_ROWS" + echo "- Unique normalized relays: $UNIQUE_RELAYS" + echo "- SHA-256: $DATA_SHA256" + echo + echo "The candidate passed strict UTF-8, schema, size, row-count, secure-host, coordinate, duplicate-conflict, and baseline-delta validation." + echo + echo "This PR is intentionally not auto-merged. Review the relay additions/removals before merging." + } > "$body_file" + + existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty') + pr_error="$RUNNER_TEMP/georelay-pr-error.txt" + pr_url="" + if [[ -n "$existing_pr" ]]; then + if gh pr edit "$existing_pr" --repo "$GITHUB_REPOSITORY" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"; then + pr_url=$(gh pr view "$existing_pr" --repo "$GITHUB_REPOSITORY" --json url --jq .url) + fi + else + if created_pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$UPDATE_BRANCH" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"); then + pr_url="$created_pr_url" + fi + fi + + tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number") + tracking_issues=() + if [[ -n "$tracking_issue_numbers" ]]; then + mapfile -t tracking_issues <<< "$tracking_issue_numbers" + fi + + if [[ -n "$pr_url" ]]; then + for issue_number in "${tracking_issues[@]}"; do + gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "A pull request is now available at $pr_url; closing this fallback tracking issue." + done + echo "Published GeoRelay review PR: $pr_url" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "::warning::GITHUB_TOKEN could not create or update the GeoRelay pull request; publishing the issues-write fallback." + if [[ -s "$pr_error" ]]; then + cat "$pr_error" >&2 + fi + + compare_url="https://github.com/${GITHUB_REPOSITORY}/compare/main...${UPDATE_BRANCH}?expand=1" + issue_body_file="$RUNNER_TEMP/georelay-tracking-issue-body.md" + { + echo "## Validated GeoRelay update awaiting review" + echo + echo "The automation branch was updated, but this workflow token could not create or update the pull request. Use the compare link below to create it manually." + echo + echo "- Compare and create PR: $compare_url" + echo "- Automation branch: $UPDATE_BRANCH" + echo "- Source: $SOURCE_URL" + echo "- Upstream commit: $SOURCE_COMMIT" + echo "- Data rows: $DATA_ROWS" + echo "- Unique normalized relays: $UNIQUE_RELAYS" + echo "- SHA-256: $DATA_SHA256" + echo + echo "The snapshot passed the repository's strict validator before the branch was pushed." + } > "$issue_body_file" + + if (( ${#tracking_issues[@]} > 0 )); then + primary_issue="${tracking_issues[0]}" + gh issue edit "$primary_issue" --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file" + issue_url=$(gh issue view "$primary_issue" --repo "$GITHUB_REPOSITORY" --json url --jq .url) + for duplicate_issue in "${tracking_issues[@]:1}"; do + gh issue close "$duplicate_issue" --repo "$GITHUB_REPOSITORY" --comment "Closing duplicate GeoRelay automation tracking issue; #$primary_issue is canonical." + done + else + issue_url=$(gh issue create --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file") + fi + + # Do not claim success until the fallback issue was confirmed. + [[ -n "$issue_url" ]] + echo "Published GeoRelay tracking issue fallback: $issue_url" >> "$GITHUB_STEP_SUMMARY" + + - name: Clean obsolete automation review state + if: steps.changes.outputs.changed == 'false' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh auth setup-git + + existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty') + if [[ -n "$existing_pr" ]]; then + gh pr close "$existing_pr" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation proposal." + echo "Closed obsolete PR #$existing_pr." >> "$GITHUB_STEP_SUMMARY" + fi + + tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number") + if [[ -n "$tracking_issue_numbers" ]]; then + while IFS= read -r issue_number; do + gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation tracker." + echo "Closed obsolete tracking issue #$issue_number." >> "$GITHUB_STEP_SUMMARY" + done <<< "$tracking_issue_numbers" + fi + + if git ls-remote --exit-code --heads origin "refs/heads/$UPDATE_BRANCH" > /dev/null; then + git push origin --delete "$UPDATE_BRANCH" + echo "Deleted obsolete automation branch $UPDATE_BRANCH." >> "$GITHUB_STEP_SUMMARY" + else + ls_remote_status=$? + if (( ls_remote_status != 2 )); then + echo "::error::Could not inspect the obsolete automation branch" + exit "$ls_remote_status" + fi + fi diff --git a/bitchat/Nostr/GeoRelayDirectory.swift b/bitchat/Nostr/GeoRelayDirectory.swift index 55a25e4d..2f4b20c2 100644 --- a/bitchat/Nostr/GeoRelayDirectory.swift +++ b/bitchat/Nostr/GeoRelayDirectory.swift @@ -32,6 +32,23 @@ struct GeoRelayDirectoryDependencies { var retrySleep: (TimeInterval) async -> Void var activeNotificationName: Notification.Name? var autoStart: Bool + var validationPolicy: GeoRelayDirectoryValidationPolicy +} + +struct GeoRelayDirectoryValidationPolicy: Sendable { + let maximumBytes: Int + let maximumRows: Int + let maximumEntries: Int + let minimumRemoteEntries: Int + let minimumRetainedFraction: Double + + static let live = GeoRelayDirectoryValidationPolicy( + maximumBytes: 512 * 1024, + maximumRows: 5_000, + maximumEntries: 5_000, + minimumRemoteEntries: 50, + minimumRetainedFraction: 0.5 + ) } private extension GeoRelayDirectoryDependencies { @@ -44,12 +61,16 @@ private extension GeoRelayDirectoryDependencies { #else let activeNotificationName: Notification.Name? = nil #endif + let validationPolicy = GeoRelayDirectoryValidationPolicy.live return Self( userDefaults: .standard, notificationCenter: .default, now: Date.init, - remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv")!, + // Runtime refreshes only from bitchat's reviewed copy. Upstream + // georelays/main is imported by a validator-backed pull request, + // so an upstream mutation cannot immediately retarget clients. + remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/bitchat/refs/heads/main/relays/online_relays_gps.csv")!, fetchInterval: TransportConfig.geoRelayFetchIntervalSeconds, refreshCheckInterval: TransportConfig.geoRelayRefreshCheckIntervalSeconds, retryInitialSeconds: TransportConfig.geoRelayRetryInitialSeconds, @@ -58,7 +79,27 @@ private extension GeoRelayDirectoryDependencies { makeFetchData: { let session = TorURLSession.shared.session return { request in - let (data, _) = try await session.data(for: request) + let (bytes, response) = try await session.bytes(for: request) + guard let response = response as? HTTPURLResponse, + (200...299).contains(response.statusCode), + response.url == request.url else { + throw URLError(.badServerResponse) + } + + let maximumBytes = validationPolicy.maximumBytes + guard response.expectedContentLength <= Int64(maximumBytes) else { + throw URLError(.dataLengthExceedsMaximum) + } + var data = Data() + if response.expectedContentLength > 0 { + data.reserveCapacity(Int(response.expectedContentLength)) + } + for try await byte in bytes { + guard data.count < maximumBytes else { + throw URLError(.dataLengthExceedsMaximum) + } + data.append(byte) + } return data } }, @@ -76,7 +117,11 @@ private extension GeoRelayDirectoryDependencies { ) let dir = base.appendingPathComponent("bitchat", isDirectory: true) try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - return dir.appendingPathComponent("georelays_cache.csv") + // v2 ignores caches populated from the old direct-upstream + // trust path and subjects every load to strict validation. + let legacyCache = dir.appendingPathComponent("georelays_cache.csv") + try? FileManager.default.removeItem(at: legacyCache) + return dir.appendingPathComponent("georelays_cache_v2.csv") } catch { return nil } @@ -94,7 +139,8 @@ private extension GeoRelayDirectoryDependencies { try? await Task.sleep(nanoseconds: nanoseconds) }, activeNotificationName: activeNotificationName, - autoStart: true + autoStart: true, + validationPolicy: validationPolicy ) } } @@ -125,7 +171,7 @@ final class GeoRelayDirectory { } private enum DetachedFetchOutcome: Sendable { - case success(entries: [Entry], csv: String) + case success(entries: [Entry], csv: Data) case torNotReady case invalidData case network(String) @@ -212,6 +258,8 @@ final class GeoRelayDirectory { ) let awaitTorReady = dependencies.awaitTorReady let fetchData = dependencies.makeFetchData() + let validationPolicy = dependencies.validationPolicy + let baselineEntries = Set(entries) Task { [weak self] in guard let self else { return } @@ -219,7 +267,9 @@ final class GeoRelayDirectory { let outcome = await Self.fetchRemoteOutcome( request: request, awaitTorReady: awaitTorReady, - fetchData: fetchData + fetchData: fetchData, + validationPolicy: validationPolicy, + baselineEntries: baselineEntries ) switch outcome { @@ -238,7 +288,9 @@ final class GeoRelayDirectory { nonisolated private static func fetchRemoteOutcome( request: URLRequest, awaitTorReady: @escaping @Sendable () async -> Bool, - fetchData: @escaping @Sendable (URLRequest) async throws -> Data + fetchData: @escaping @Sendable (URLRequest) async throws -> Data, + validationPolicy: GeoRelayDirectoryValidationPolicy, + baselineEntries: Set ) async -> DetachedFetchOutcome { await Task.detached(priority: .utility) { let ready = await awaitTorReady() @@ -246,16 +298,16 @@ final class GeoRelayDirectory { do { let data = try await fetchData(request) - guard let text = String(data: data, encoding: .utf8) else { + guard let parsed = Self.validatedEntries( + from: data, + policy: validationPolicy, + minimumEntries: validationPolicy.minimumRemoteEntries, + baselineEntries: baselineEntries + ) else { return .invalidData } - let parsed = Self.parseCSV(text) - guard !parsed.isEmpty else { - return .invalidData - } - - return .success(entries: parsed, csv: text) + return .success(entries: parsed, csv: data) } catch { return .network(error.localizedDescription) } @@ -269,7 +321,7 @@ final class GeoRelayDirectory { } @MainActor - private func handleFetchSuccess(entries parsed: [Entry], csv: String) { + private func handleFetchSuccess(entries parsed: [Entry], csv: Data) { entries = parsed persistCache(csv) dependencies.userDefaults.set(dependencies.now(), forKey: lastFetchKey) @@ -321,9 +373,8 @@ final class GeoRelayDirectory { cleanupState.retryTask = nil } - private func persistCache(_ text: String) { + private func persistCache(_ data: Data) { guard let url = dependencies.cacheURL() else { return } - guard let data = text.data(using: .utf8) else { return } do { try dependencies.writeData(data, url) } catch { @@ -336,9 +387,12 @@ final class GeoRelayDirectory { // Prefer cached file if present if let cache = dependencies.cacheURL(), let data = dependencies.readData(cache), - let text = String(data: data, encoding: .utf8) { - let arr = Self.parseCSV(text) - if !arr.isEmpty { return arr } + let entries = Self.validatedEntries( + from: data, + policy: dependencies.validationPolicy, + minimumEntries: 1 + ) { + return entries } // Try bundled resource(s) @@ -346,36 +400,157 @@ final class GeoRelayDirectory { for url in bundleCandidates { if let data = dependencies.readData(url), - let text = String(data: data, encoding: .utf8) { - let arr = Self.parseCSV(text) - if !arr.isEmpty { return arr } + let entries = Self.validatedEntries( + from: data, + policy: dependencies.validationPolicy, + minimumEntries: 1 + ) { + return entries } } // Try filesystem path (development/test) if let cwd = dependencies.currentDirectoryPath(), let data = dependencies.readData(URL(fileURLWithPath: cwd).appendingPathComponent("relays/online_relays_gps.csv")), - let text = String(data: data, encoding: .utf8) { - return Self.parseCSV(text) + let entries = Self.validatedEntries( + from: data, + policy: dependencies.validationPolicy, + minimumEntries: 1 + ) { + return entries } SecureLogger.warning("GeoRelayDirectory: no local CSV found; entries empty", category: .session) return [] } - nonisolated static func parseCSV(_ text: String) -> [Entry] { - var result: Set = [] - let lines = text.split(whereSeparator: { $0.isNewline }) - for (idx, raw) in lines.enumerated() { - guard let line = raw.trimmedOrNilIfEmpty else { continue } - if idx == 0 && line.lowercased().contains("relay url") { continue } - let parts = line.split(separator: ",").map { $0.trimmed } - guard parts.count >= 3 else { continue } - guard let host = NostrRelayURL.directoryAddress(parts[0]) else { continue } - guard let lat = Double(parts[1]), let lon = Double(parts[2]) else { continue } - result.insert(Entry(host: host, lat: lat, lon: lon)) + /// Parses the fixed three-column format as an all-or-nothing trust unit. + /// One malformed or conflicting row rejects the complete dataset rather + /// than silently shrinking or partially replacing the current directory. + nonisolated static func validatedEntries( + from data: Data, + policy: GeoRelayDirectoryValidationPolicy, + minimumEntries: Int, + baselineEntries: Set? = nil + ) -> [Entry]? { + guard !data.isEmpty, data.count <= policy.maximumBytes, + let text = String(data: data, encoding: .utf8), + !text.hasPrefix("\u{feff}") else { + return nil } - return Array(result) + + let lines = text.split(whereSeparator: { $0.isNewline }) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard let header = lines.first, + lines.count - 1 <= policy.maximumRows else { + return nil + } + + let headerParts = header + .split(separator: ",", omittingEmptySubsequences: false) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() } + let supportedHeaders = [ + ["relay url", "latitude", "longitude"], + ["relay url", "lat", "lon"] + ] + guard supportedHeaders.contains(headerParts) else { + return nil + } + + var entriesByHost: [String: Entry] = [:] + for line in lines.dropFirst() { + let parts = line + .split(separator: ",", omittingEmptySubsequences: false) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + guard parts.count == 3, + let host = validatedDirectoryAddress(parts[0]), + let latitude = Double(parts[1]), latitude.isFinite, + (-90.0...90.0).contains(latitude), + let longitude = Double(parts[2]), longitude.isFinite, + (-180.0...180.0).contains(longitude) else { + return nil + } + + let entry = Entry(host: host, lat: latitude, lon: longitude) + if let existing = entriesByHost[host], existing != entry { + // One endpoint cannot truthfully occupy two coordinates. Do + // not let row ordering choose which location clients trust. + return nil + } + entriesByHost[host] = entry + guard entriesByHost.count <= policy.maximumEntries else { return nil } + } + + let parsedEntries = Set(entriesByHost.values) + guard parsedEntries.count >= minimumEntries else { return nil } + + if let baselineEntries { + guard (0...1).contains(policy.minimumRetainedFraction) else { return nil } + let requiredOverlap = Int( + ceil(Double(baselineEntries.count) * policy.minimumRetainedFraction) + ) + guard parsedEntries.intersection(baselineEntries).count >= requiredOverlap else { + return nil + } + } + + return parsedEntries.sorted { + ($0.host, $0.lat, $0.lon) < ($1.host, $1.lat, $1.lon) + } + } + + nonisolated private static func validatedDirectoryAddress(_ rawValue: String) -> String? { + let value = rawValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, + value.unicodeScalars.allSatisfy({ + $0.isASCII && !CharacterSet.controlCharacters.contains($0) + }) else { + return nil + } + + let candidate = value.contains("://") ? value : "wss://\(value)" + guard let components = URLComponents(string: candidate), + let scheme = components.scheme?.lowercased(), + scheme == "wss" || scheme == "https", + components.user == nil, + components.password == nil, + components.query == nil, + components.fragment == nil, + components.path.isEmpty || components.path == "/", + let rawHost = components.host else { + return nil + } + + let host = rawHost.lowercased() + guard !host.isEmpty, host.count <= 253, + host.unicodeScalars.allSatisfy({ $0.isASCII }), + !host.hasSuffix("."), + host != "localhost", + !host.hasSuffix(".localhost"), + !host.hasSuffix(".local"), + !host.hasSuffix(".internal") else { + return nil + } + + let labels = host.split(separator: ".", omittingEmptySubsequences: false) + let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyz0123456789-") + guard labels.count >= 2, + !labels.allSatisfy({ $0.allSatisfy(\.isNumber) }), + labels.allSatisfy({ label in + (1...63).contains(label.count) && + label.first != "-" && + label.last != "-" && + label.unicodeScalars.allSatisfy { allowed.contains($0) } + }) else { + return nil + } + + if let port = components.port { + guard (1...65_535).contains(port) else { return nil } + if port != 443 { return "\(host):\(port)" } + } + return host } // MARK: - Observers & Timers diff --git a/bitchat/Nostr/NostrRelayURL.swift b/bitchat/Nostr/NostrRelayURL.swift index be9a6fb6..eb2e2afc 100644 --- a/bitchat/Nostr/NostrRelayURL.swift +++ b/bitchat/Nostr/NostrRelayURL.swift @@ -39,13 +39,4 @@ enum NostrRelayURL { return components.string } - - static func directoryAddress(_ rawValue: String) -> String? { - guard var normalized = normalized(rawValue, defaultScheme: "wss") else { return nil } - for prefix in ["wss://", "ws://"] where normalized.hasPrefix(prefix) { - normalized.removeFirst(prefix.count) - break - } - return normalized - } } diff --git a/bitchatTests/Nostr/GeoRelayDirectoryTests.swift b/bitchatTests/Nostr/GeoRelayDirectoryTests.swift index 516f4a1d..bbcccc42 100644 --- a/bitchatTests/Nostr/GeoRelayDirectoryTests.swift +++ b/bitchatTests/Nostr/GeoRelayDirectoryTests.swift @@ -5,19 +5,25 @@ import XCTest @MainActor final class GeoRelayDirectoryTests: XCTestCase { - func test_parseCSV_normalizesRelaySchemesAndDeduplicatesEntries() { + private func parse(_ csv: String) -> [GeoRelayDirectory.Entry] { + GeoRelayDirectory.validatedEntries( + from: Data(csv.utf8), + policy: .live, + minimumEntries: 1 + ) ?? [] + } + + func test_parseCSV_normalizesSecureRelaySchemesAndDeduplicatesEntries() { let csv = """ relay url,lat,lon wss://one.example/,10,20 https://one.example,10,20 wss://one.example:443/,10,20 - http://two.example/,11,21 + two.example,11,21 wss://two.example:443,11,21 - invalid row - ws://three.example,not-a-lat,22 """ - let parsed = Set(GeoRelayDirectory.parseCSV(csv)) + let parsed = Set(parse(csv)) XCTAssertEqual( parsed, @@ -28,6 +34,136 @@ final class GeoRelayDirectoryTests: XCTestCase { ) } + func test_parseCSV_rejectsWholeDatasetWhenAnyRowOrHeaderIsUnsafe() { + let invalidCSVs = [ + "relay,lat,lon\nrelay.example,1,2\n", + "relay url,lat,lon\nrelay.example,1\n", + "relay url,lat,lon\nhttp://relay.example,1,2\n", + "relay url,lat,lon\nwss://user@relay.example,1,2\n", + "relay url,lat,lon\nwss://relay.example/path,1,2\n", + "relay url,lat,lon\nwss://relay.example?,1,2\n", + "relay url,lat,lon\nwss://relay.example#,1,2\n", + "relay url,lat,lon\nrelay.example:0,1,2\n", + "relay url,lat,lon\nrelay.example:99999,1,2\n", + "relay url,lat,lon\nlocalhost,1,2\n", + "relay url,lat,lon\nr\u{00e9}lay.example,1,2\n", + "relay url,lat,lon\nrelay\u{202e}.example,1,2\n", + "relay url,lat,lon\nrelay.example,NaN,2\n", + "relay url,lat,lon\nrelay.example,1_0,2\n", + "relay url,lat,lon\nrelay.example,\u{0661}\u{0660},2\n", + "relay url,lat,lon\nrelay.example,\u{ff11}\u{ff10},2\n", + "relay url,lat,lon\nrelay.example,91,2\n", + "relay url,lat,lon\nrelay.example,1,181\n", + "relay url,lat,lon\nrelay.example,1,2\nrelay.example,3,4\n" + ] + + for csv in invalidCSVs { + XCTAssertTrue(parse(csv).isEmpty, csv) + } + } + + func test_validatedEntries_enforcesByteRowEntryAndRetentionLimits() { + let restrictive = GeoRelayDirectoryValidationPolicy( + maximumBytes: 100, + maximumRows: 2, + maximumEntries: 2, + minimumRemoteEntries: 1, + minimumRetainedFraction: 0.5 + ) + let one = Data("relay url,lat,lon\none.example,1,2\n".utf8) + let three = Data("relay url,lat,lon\none.example,1,2\ntwo.example,3,4\nthree.example,5,6\n".utf8) + + XCTAssertNil(GeoRelayDirectory.validatedEntries( + from: one, + policy: restrictive, + minimumEntries: 2 + )) + XCTAssertNil(GeoRelayDirectory.validatedEntries( + from: Data(repeating: 0x41, count: 101), + policy: restrictive, + minimumEntries: 1 + )) + XCTAssertNil(GeoRelayDirectory.validatedEntries( + from: three, + policy: restrictive, + minimumEntries: 1 + )) + } + + func test_validatedEntries_requiresExactBaselineEntryOverlap() throws { + let policy = GeoRelayDirectoryValidationPolicy( + maximumBytes: 1_000, + maximumRows: 10, + maximumEntries: 10, + minimumRemoteEntries: 1, + minimumRetainedFraction: 0.5 + ) + let baseline = Set(try XCTUnwrap(GeoRelayDirectory.validatedEntries( + from: Data(""" + relay url,lat,lon + one.example,1,1 + two.example,2,2 + three.example,3,3 + """.utf8), + policy: policy, + minimumEntries: 1 + ))) + let disjoint = Data(""" + relay url,lat,lon + four.example,1,1 + five.example,2,2 + six.example,3,3 + """.utf8) + let rewrittenCoordinates = Data(""" + relay url,lat,lon + one.example,11,11 + two.example,12,12 + three.example,13,13 + """.utf8) + let halfRetained = Data(""" + relay url,lat,lon + wss://one.example:443/,1,1 + https://two.example/,2,2 + replacement.example,4,4 + """.utf8) + + XCTAssertNil(GeoRelayDirectory.validatedEntries( + from: disjoint, + policy: policy, + minimumEntries: 1, + baselineEntries: baseline + )) + XCTAssertNil(GeoRelayDirectory.validatedEntries( + from: rewrittenCoordinates, + policy: policy, + minimumEntries: 1, + baselineEntries: baseline + )) + XCTAssertNotNil(GeoRelayDirectory.validatedEntries( + from: halfRetained, + policy: policy, + minimumEntries: 1, + baselineEntries: baseline + )) + } + + func test_bundledReviewedCSV_passesStrictProductionValidation() throws { + let repositoryRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let data = try Data( + contentsOf: repositoryRoot.appendingPathComponent("relays/online_relays_gps.csv") + ) + + let entries = try XCTUnwrap(GeoRelayDirectory.validatedEntries( + from: data, + policy: .live, + minimumEntries: GeoRelayDirectoryValidationPolicy.live.minimumRemoteEntries + )) + XCTAssertGreaterThan(entries.count, 250) + } + func test_closestRelays_sortsByDistanceForLatLonAndGeohash() { let harness = makeHarness( cacheCSV: """ @@ -243,6 +379,53 @@ final class GeoRelayDirectoryTests: XCTestCase { XCTAssertFalse(directory.debugHasRetryTask) } + func test_prefetchIfNeeded_rejectsSharpValidLookingTruncationBeforeCaching() async { + let cached = """ + relay url,lat,lon + old-one.example,1,1 + old-two.example,2,2 + old-three.example,3,3 + """ + let truncated = """ + relay url,lat,lon + attacker.example,9,9 + """ + let recovered = """ + relay url,lat,lon + old-one.example,1,1 + old-two.example,2,2 + new-three.example,6,6 + """ + let harness = makeHarness( + cacheCSV: cached, + fetchResults: [ + .success(Data(truncated.utf8)), + .success(Data(recovered.utf8)) + ], + validationPolicy: GeoRelayDirectoryValidationPolicy( + maximumBytes: 64 * 1024, + maximumRows: 1_000, + maximumEntries: 1_000, + minimumRemoteEntries: 1, + minimumRetainedFraction: 0.5 + ) + ) + let directory = GeoRelayDirectory(dependencies: harness.dependencies) + + directory.prefetchIfNeeded() + + let refreshed = await waitUntil { + directory.entries.contains(where: { $0.host == "new-three.example" }) + } + XCTAssertTrue(refreshed) + XCTAssertFalse(directory.entries.contains(where: { $0.host == "attacker.example" })) + let requestCount = await harness.fetcher.recordedRequestCount() + let retryDelays = await harness.retryRecorder.recordedDelays() + XCTAssertEqual(requestCount, 2) + XCTAssertEqual(retryDelays, [5]) + XCTAssertEqual(harness.fileStore.dataByURL[harness.cacheURL], Data(recovered.utf8)) + } + func test_observers_triggerPrefetchesForTorReadyAndAppActivation() async { let activeNotification = Notification.Name("GeoRelayDirectoryTests.didBecomeActive") let harness = makeHarness( @@ -289,7 +472,14 @@ final class GeoRelayDirectoryTests: XCTestCase { fetchFactoryObserver: (@MainActor @Sendable () -> Void)? = nil, fetchObserver: (@Sendable () async -> Void)? = nil, autoStart: Bool = false, - activeNotificationName: Notification.Name? = nil + activeNotificationName: Notification.Name? = nil, + validationPolicy: GeoRelayDirectoryValidationPolicy = GeoRelayDirectoryValidationPolicy( + maximumBytes: 64 * 1024, + maximumRows: 1_000, + maximumEntries: 1_000, + minimumRemoteEntries: 1, + minimumRetainedFraction: 0 + ) ) -> GeoRelayHarness { let userDefaultsSuite = "GeoRelayDirectoryTests.\(UUID().uuidString)" let userDefaults = UserDefaults(suiteName: userDefaultsSuite)! @@ -347,7 +537,8 @@ final class GeoRelayDirectoryTests: XCTestCase { await retryRecorder.record(delay) }, activeNotificationName: activeNotificationName, - autoStart: autoStart + autoStart: autoStart, + validationPolicy: validationPolicy ) return GeoRelayHarness( diff --git a/scripts/tests/test_fetch_georelays_workflow.py b/scripts/tests/test_fetch_georelays_workflow.py new file mode 100644 index 00000000..57c5d81d --- /dev/null +++ b/scripts/tests/test_fetch_georelays_workflow.py @@ -0,0 +1,61 @@ +import re +from pathlib import Path +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW_PATH = REPOSITORY_ROOT / ".github/workflows/fetch_georelays.yml" + + +class FetchGeoRelaysWorkflowTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + + def test_write_capable_checkout_action_is_immutable(self) -> None: + checkout = re.search(r"uses: actions/checkout@([0-9a-f]+)", self.workflow) + self.assertIsNotNone(checkout) + self.assertRegex(checkout.group(1), r"^[0-9a-f]{40}$") + self.assertIn("persist-credentials: false", self.workflow) + + def test_pr_failure_has_single_issue_fallback_with_review_metadata(self) -> None: + required_fragments = [ + "issues: write", + "TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request", + "gh pr create", + "gh issue create", + "gh issue edit", + "compare/main...${UPDATE_BRANCH}?expand=1", + "Upstream commit: $SOURCE_COMMIT", + "Data rows: $DATA_ROWS", + "Unique normalized relays: $UNIQUE_RELAYS", + "SHA-256: $DATA_SHA256", + '[[ -n "$issue_url" ]]', + ] + for fragment in required_fragments: + with self.subTest(fragment=fragment): + self.assertIn(fragment, self.workflow) + + confirmed = self.workflow.index('[[ -n "$issue_url" ]]') + success_summary = self.workflow.index( + "Published GeoRelay tracking issue fallback: $issue_url" + ) + self.assertLess(confirmed, success_summary) + + def test_obsolete_review_state_is_cleaned_without_pushing_main(self) -> None: + self.assertIn("gh pr close", self.workflow) + self.assertIn("gh issue close", self.workflow) + self.assertIn('git push origin --delete "$UPDATE_BRANCH"', self.workflow) + self.assertIn('git switch -C "$UPDATE_BRANCH"', self.workflow) + self.assertNotIn("git push origin main", self.workflow) + self.assertNotIn("git push --force origin main", self.workflow) + + def test_workflow_runs_all_validator_tests(self) -> None: + self.assertIn( + 'python3 -m unittest discover -s scripts/tests -p "test_*.py" -v', + self.workflow, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_validate_georelays.py b/scripts/tests/test_validate_georelays.py new file mode 100644 index 00000000..faf63834 --- /dev/null +++ b/scripts/tests/test_validate_georelays.py @@ -0,0 +1,186 @@ +import tempfile +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import validate_georelays as validator + + +def csv_bytes(rows: list[str]) -> bytes: + return ("Relay URL,Latitude,Longitude\n" + "\n".join(rows) + "\n").encode() + + +class ValidateGeoRelaysTests(unittest.TestCase): + def test_validates_and_deduplicates_secure_relay_addresses(self) -> None: + data = csv_bytes( + [ + "relay.example.com,10,20", + "wss://relay.example.com:443/,10,20", + "https://second.example.org,11,21", + ] + ) + + summary = validator.validate_bytes(data, minimum_unique_relays=2) + + self.assertEqual(summary.data_rows, 3) + self.assertEqual(summary.unique_relays, 2) + + def test_rejects_insecure_or_non_host_relay_urls(self) -> None: + bad_addresses = [ + "http://relay.example.com", + "ws://relay.example.com", + "wss://user@relay.example.com", + "wss://relay.example.com/path", + "wss://relay.example.com?", + "wss://relay.example.com#", + "relay.example.com:0", + "relay.example.com:99999", + "localhost", + "127.0.0.1", + "relay_example.com", + "relay\u202e.example.com", + ] + + for address in bad_addresses: + with self.subTest(address=address): + with self.assertRaises(validator.ValidationError): + validator.validate_bytes( + csv_bytes([f"{address},10,20"]), + minimum_unique_relays=1, + ) + + def test_rejects_malformed_rows_and_unsafe_coordinates(self) -> None: + bad_rows = [ + "relay.example.com,10", + "relay.example.com,NaN,20", + "relay.example.com,1_0,20", + "relay.example.com,\u0661\u0660,20", + "relay.example.com,\uff11\uff10,20", + "relay.example.com,91,20", + "relay.example.com,10,-181", + "relay.example.com,10,20,extra", + '"relay.example.com",10,20', + ] + + for row in bad_rows: + with self.subTest(row=row): + with self.assertRaises(validator.ValidationError): + validator.validate_bytes(csv_bytes([row]), minimum_unique_relays=1) + + def test_accepts_ascii_coordinate_forms_supported_by_swift_double(self) -> None: + summary = validator.validate_bytes( + csv_bytes( + [ + "one.example.com,+1,-.5", + "two.example.com,1.e1,2E+1", + "three.example.com,01,20.", + ] + ), + minimum_unique_relays=3, + ) + + self.assertEqual(summary.unique_relays, 3) + + def test_rejects_conflicts_limits_and_large_baseline_deltas(self) -> None: + with self.assertRaises(validator.ValidationError): + validator.validate_bytes( + csv_bytes(["relay.example.com,10,20", "relay.example.com,11,21"]), + minimum_unique_relays=1, + ) + with self.assertRaises(validator.ValidationError): + validator.validate_bytes(b"x" * 20, maximum_bytes=10, minimum_unique_relays=1) + with self.assertRaises(validator.ValidationError): + validator.validate_bytes( + csv_bytes(["one.example.com,1,1", "two.example.com,2,2"]), + minimum_unique_relays=3, + ) + + baseline = csv_bytes( + [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(120)] + ) + shrunken = csv_bytes( + [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(59)] + ) + with self.assertRaises(validator.ValidationError): + validator.validate_update(shrunken, baseline) + + smaller_baseline = csv_bytes( + [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)] + ) + expanded = csv_bytes( + [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(121)] + ) + with self.assertRaises(validator.ValidationError): + validator.validate_update(expanded, smaller_baseline) + + def test_update_requires_exact_normalized_baseline_entry_overlap(self) -> None: + baseline_rows = [ + f"relay-{index}.example.com,{index % 80},{index % 170}" + for index in range(60) + ] + baseline = csv_bytes(baseline_rows) + disjoint = csv_bytes( + [ + f"attacker-{index}.example.com,{index % 80},{index % 170}" + for index in range(60) + ] + ) + rewritten_coordinates = csv_bytes( + [ + f"relay-{index}.example.com,{(index % 80) + 0.5},{index % 170}" + for index in range(60) + ] + ) + + for candidate in (disjoint, rewritten_coordinates): + with self.subTest(candidate=candidate[:80]): + with self.assertRaisesRegex( + validator.ValidationError, + "exact relay-coordinate entries", + ): + validator.validate_update(candidate, baseline) + + half_retained = csv_bytes( + [ + f"wss://relay-{index}.example.com:443/,{index % 80},{index % 170}" + for index in range(30) + ] + + [ + f"replacement-{index}.example.com,{index % 80},{index % 170}" + for index in range(30) + ] + ) + summary = validator.validate_update(half_retained, baseline) + self.assertEqual(summary.unique_relays, 60) + + def test_cli_copies_only_validated_data_and_emits_review_metadata(self) -> None: + rows = [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)] + data = csv_bytes(rows) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate.csv" + baseline = root / "baseline.csv" + output = root / "output.csv" + github_output = root / "github-output.txt" + candidate.write_bytes(data) + baseline.write_bytes(data) + + result = validator.main( + [ + "--input", str(candidate), + "--baseline", str(baseline), + "--output", str(output), + "--github-output", str(github_output), + ] + ) + + self.assertEqual(result, 0) + self.assertEqual(output.read_bytes(), data) + metadata = github_output.read_text() + self.assertIn("unique_relays=60", metadata) + self.assertIn("sha256=", metadata) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_georelays.py b/scripts/validate_georelays.py new file mode 100644 index 00000000..513a4dc0 --- /dev/null +++ b/scripts/validate_georelays.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Strict validator for the reviewed georelay CSV update workflow.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import math +import re +from dataclasses import dataclass +from pathlib import Path +import sys +import unicodedata +from urllib.parse import urlsplit + + +MAX_BYTES = 512 * 1024 +MAX_ROWS = 5_000 +MAX_UNIQUE_RELAYS = 5_000 +MIN_UNIQUE_RELAYS = 50 +MIN_BASELINE_FRACTION = 0.5 +MAX_BASELINE_MULTIPLIER = 2.0 +EXPECTED_HEADER = ("relay url", "latitude", "longitude") +ASCII_DECIMAL_PATTERN = re.compile( + r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z" +) + + +class ValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class ValidationSummary: + data_rows: int + unique_relays: int + sha256: str + + +@dataclass(frozen=True) +class _ValidatedDataset: + summary: ValidationSummary + entries: frozenset[tuple[str, float, float]] + + +def _has_disallowed_control(value: str) -> bool: + return any( + unicodedata.category(character) in {"Cc", "Cf"} + and character not in {"\r", "\n", "\t"} + for character in value + ) + + +def normalize_relay_address(raw_value: str) -> str: + value = raw_value.strip() + if not value or _has_disallowed_control(value): + raise ValidationError("relay address is empty or contains control characters") + # urlsplit cannot distinguish an absent query/fragment from an explicitly + # empty one. Reject the delimiters themselves so this validator matches + # URLComponents in the client and reviewed data cannot fail closed there. + if "?" in value or "#" in value: + raise ValidationError(f"relay query or fragment is not allowed: {value}") + + candidate = value if "://" in value else f"wss://{value}" + try: + parsed = urlsplit(candidate) + port = parsed.port + except ValueError as error: + raise ValidationError(f"invalid relay URL: {value}") from error + + if parsed.scheme.lower() not in {"wss", "https"}: + raise ValidationError(f"relay must use wss/https or a bare hostname: {value}") + if parsed.username is not None or parsed.password is not None: + raise ValidationError(f"relay credentials are not allowed: {value}") + if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: + raise ValidationError(f"relay path, query, or fragment is not allowed: {value}") + + host = (parsed.hostname or "").lower() + if not host or len(host) > 253 or not host.isascii(): + raise ValidationError(f"relay hostname is missing or non-ASCII: {value}") + if host.endswith(".") or host == "localhost" or host.endswith((".localhost", ".local", ".internal")): + raise ValidationError(f"local or absolute relay hostname is not allowed: {value}") + + labels = host.split(".") + if len(labels) < 2 or all(label.isdigit() for label in labels): + raise ValidationError(f"relay must use a public DNS hostname: {value}") + for label in labels: + if not 1 <= len(label) <= 63: + raise ValidationError(f"invalid DNS label length: {value}") + if label[0] == "-" or label[-1] == "-": + raise ValidationError(f"DNS labels cannot start or end with '-': {value}") + if any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label): + raise ValidationError(f"invalid DNS hostname character: {value}") + + if port is not None and not 1 <= port <= 65_535: + raise ValidationError(f"invalid relay port: {value}") + if port in {None, 443}: + return host + return f"{host}:{port}" + + +def _validated_dataset( + data: bytes, + *, + minimum_unique_relays: int = MIN_UNIQUE_RELAYS, + maximum_bytes: int = MAX_BYTES, + maximum_rows: int = MAX_ROWS, + maximum_unique_relays: int = MAX_UNIQUE_RELAYS, +) -> _ValidatedDataset: + if not data or len(data) > maximum_bytes: + raise ValidationError(f"CSV must contain 1..{maximum_bytes} bytes") + + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + raise ValidationError("CSV is not valid UTF-8") from error + if text.startswith("\ufeff"): + raise ValidationError("UTF-8 BOM is not allowed") + if _has_disallowed_control(text): + raise ValidationError("CSV contains disallowed control characters") + # Runtime intentionally implements the fixed three-field schema without + # general CSV quoting. Reject quoted variants here so reviewed workflow + # output and client-side validation cannot disagree. + if '"' in text: + raise ValidationError("quoted CSV fields are not allowed") + + reader = csv.reader(io.StringIO(text, newline=""), strict=True) + try: + header = next(reader) + except (StopIteration, csv.Error) as error: + raise ValidationError("CSV header is missing") from error + normalized_header = tuple(field.strip().lower() for field in header) + if normalized_header != EXPECTED_HEADER: + raise ValidationError(f"unexpected CSV header: {header!r}") + + data_rows = 0 + relays: dict[str, tuple[float, float]] = {} + try: + for row in reader: + if not row or all(not field.strip() for field in row): + continue + data_rows += 1 + if data_rows > maximum_rows: + raise ValidationError(f"CSV exceeds {maximum_rows} data rows") + if len(row) != 3: + raise ValidationError(f"row {reader.line_num} must contain exactly 3 columns") + + address = normalize_relay_address(row[0]) + latitude_text = row[1].strip() + longitude_text = row[2].strip() + if not ASCII_DECIMAL_PATTERN.fullmatch(latitude_text) or not ASCII_DECIMAL_PATTERN.fullmatch(longitude_text): + raise ValidationError( + f"row {reader.line_num} coordinates must be ASCII decimal numbers" + ) + latitude = float(latitude_text) + longitude = float(longitude_text) + if not math.isfinite(latitude) or not -90 <= latitude <= 90: + raise ValidationError(f"row {reader.line_num} latitude is out of range") + if not math.isfinite(longitude) or not -180 <= longitude <= 180: + raise ValidationError(f"row {reader.line_num} longitude is out of range") + + coordinates = (latitude, longitude) + previous = relays.get(address) + if previous is not None and previous != coordinates: + raise ValidationError(f"relay {address} has conflicting coordinates") + relays[address] = coordinates + if len(relays) > maximum_unique_relays: + raise ValidationError(f"CSV exceeds {maximum_unique_relays} unique relays") + except csv.Error as error: + raise ValidationError(f"malformed CSV near line {reader.line_num}") from error + + if len(relays) < minimum_unique_relays: + raise ValidationError( + f"CSV has {len(relays)} unique relays; minimum is {minimum_unique_relays}" + ) + + return _ValidatedDataset( + summary=ValidationSummary( + data_rows=data_rows, + unique_relays=len(relays), + sha256=hashlib.sha256(data).hexdigest(), + ), + entries=frozenset( + (address, coordinates[0], coordinates[1]) + for address, coordinates in relays.items() + ), + ) + + +def validate_bytes( + data: bytes, + *, + minimum_unique_relays: int = MIN_UNIQUE_RELAYS, + maximum_bytes: int = MAX_BYTES, + maximum_rows: int = MAX_ROWS, + maximum_unique_relays: int = MAX_UNIQUE_RELAYS, +) -> ValidationSummary: + return _validated_dataset( + data, + minimum_unique_relays=minimum_unique_relays, + maximum_bytes=maximum_bytes, + maximum_rows=maximum_rows, + maximum_unique_relays=maximum_unique_relays, + ).summary + + +def validate_update(candidate: bytes, baseline: bytes) -> ValidationSummary: + baseline_dataset = _validated_dataset(baseline, minimum_unique_relays=1) + candidate_dataset = _validated_dataset(candidate) + baseline_summary = baseline_dataset.summary + candidate_summary = candidate_dataset.summary + + minimum_from_baseline = math.ceil( + baseline_summary.unique_relays * MIN_BASELINE_FRACTION + ) + maximum_from_baseline = math.floor( + baseline_summary.unique_relays * MAX_BASELINE_MULTIPLIER + ) + if candidate_summary.unique_relays < minimum_from_baseline: + raise ValidationError( + "candidate loses more than half of the baseline's unique relays " + f"({candidate_summary.unique_relays} < {minimum_from_baseline})" + ) + if candidate_summary.unique_relays > maximum_from_baseline: + raise ValidationError( + "candidate more than doubles the baseline's unique relays " + f"({candidate_summary.unique_relays} > {maximum_from_baseline})" + ) + + retained_entries = len(baseline_dataset.entries & candidate_dataset.entries) + if retained_entries < minimum_from_baseline: + raise ValidationError( + "candidate retains fewer than half of the baseline's exact relay-coordinate entries " + f"({retained_entries} < {minimum_from_baseline})" + ) + return candidate_summary + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--github-output", type=Path) + args = parser.parse_args(argv) + + try: + candidate = args.input.read_bytes() + baseline = args.baseline.read_bytes() + summary = validate_update(candidate, baseline) + args.output.write_bytes(candidate) + if args.github_output is not None: + with args.github_output.open("a", encoding="utf-8") as output: + output.write(f"data_rows={summary.data_rows}\n") + output.write(f"unique_relays={summary.unique_relays}\n") + output.write(f"sha256={summary.sha256}\n") + except (OSError, ValidationError) as error: + print(f"georelay validation failed: {error}", file=sys.stderr) + return 1 + + print( + f"validated {summary.unique_relays} unique relays across " + f"{summary.data_rows} rows (sha256 {summary.sha256})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())