mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-24 21:45:20 +00:00
Route georelay updates through review
This commit is contained in:
@@ -1,42 +1,228 @@
|
||||
name: Fetch GeoRelays Data
|
||||
name: Propose GeoRelay Data Update
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 0'
|
||||
- cron: "0 6 * * 0"
|
||||
workflow_dispatch:
|
||||
|
||||
# Default to read-only. The publishing job receives only the scopes required
|
||||
# to push its branch and publish either a PR or a tracking issue.
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: georelay-data-update
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
SOURCE_REPOSITORY: https://github.com/permissionlesstech/georelays.git
|
||||
UPDATE_BRANCH: automation/georelay-data
|
||||
TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request
|
||||
|
||||
jobs:
|
||||
update-relay-data:
|
||||
propose-relay-data:
|
||||
name: Validate and propose relay data
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Checkout reviewed base
|
||||
# Pinned actions/checkout v5 so a mutable action tag cannot change the
|
||||
# code that receives this job's write-capable token.
|
||||
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
# Do not expose the write token to fetch/validation subprocesses.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch GeoRelays
|
||||
- name: Test GeoRelay validator
|
||||
run: |
|
||||
wget -q https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv
|
||||
mv nostr_relays.csv ./relays/online_relays_gps.csv
|
||||
set -euo pipefail
|
||||
python3 -m unittest discover -s scripts/tests -p "test_*.py" -v
|
||||
|
||||
- name: Check for changes
|
||||
id: git-check
|
||||
- name: Fetch candidate over pinned HTTPS policy
|
||||
id: upstream
|
||||
run: |
|
||||
git diff --exit-code || echo "changes=true" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.git-check.outputs.changes == 'true'
|
||||
set -euo pipefail
|
||||
source_commit=$(git ls-remote --refs "$SOURCE_REPOSITORY" refs/heads/main | awk 'NR == 1 { print $1 }')
|
||||
if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo "::error::Could not resolve an immutable upstream commit"
|
||||
exit 1
|
||||
fi
|
||||
source_url="https://raw.githubusercontent.com/permissionlesstech/georelays/$source_commit/nostr_relays.csv"
|
||||
effective_url=$(curl --fail --show-error --silent --location --proto "=https" --proto-redir "=https" --tlsv1.2 --max-time 60 --retry 3 --retry-all-errors --output "$RUNNER_TEMP/georelays-candidate.csv" --write-out "%{url_effective}" "$source_url")
|
||||
if [[ "$effective_url" != "$source_url" ]]; then
|
||||
echo "::error::Unexpected GeoRelay redirect target: $effective_url"
|
||||
exit 1
|
||||
fi
|
||||
echo "source_commit=$source_commit" >> "$GITHUB_OUTPUT"
|
||||
echo "source_url=$source_url" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate candidate against reviewed baseline
|
||||
id: validation
|
||||
run: |
|
||||
git config --local user.email "action@github.com"
|
||||
git config --local user.name "GitHub Action"
|
||||
git add relays/online_relays_gps.csv
|
||||
git commit -m "Automated update of relay data - $(date -u)"
|
||||
git push
|
||||
set -euo pipefail
|
||||
python3 scripts/validate_georelays.py --input "$RUNNER_TEMP/georelays-candidate.csv" --baseline relays/online_relays_gps.csv --output relays/online_relays_gps.csv --github-output "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check for a reviewed-file change
|
||||
id: changes
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet -- relays/online_relays_gps.csv; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Upstream GeoRelay data already matches main." >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
git diff --stat -- relays/online_relays_gps.csv
|
||||
fi
|
||||
|
||||
- name: Push automation branch and publish review request
|
||||
if: steps.changes.outputs.changed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SOURCE_COMMIT: ${{ steps.upstream.outputs.source_commit }}
|
||||
SOURCE_URL: ${{ steps.upstream.outputs.source_url }}
|
||||
DATA_ROWS: ${{ steps.validation.outputs.data_rows }}
|
||||
UNIQUE_RELAYS: ${{ steps.validation.outputs.unique_relays }}
|
||||
DATA_SHA256: ${{ steps.validation.outputs.sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Scope credential exposure to this final publishing step.
|
||||
gh auth setup-git
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
git switch -C "$UPDATE_BRANCH"
|
||||
git add -- relays/online_relays_gps.csv
|
||||
git diff --cached --quiet && {
|
||||
echo "::error::Expected a staged GeoRelay data change"
|
||||
exit 1
|
||||
}
|
||||
git commit -m "Update reviewed georelay directory" -m "Upstream-commit: $SOURCE_COMMIT"
|
||||
|
||||
remote_ref="refs/remotes/origin/$UPDATE_BRANCH"
|
||||
if git fetch --no-tags origin "+refs/heads/$UPDATE_BRANCH:$remote_ref" 2>/dev/null; then
|
||||
remote_sha=$(git rev-parse "$remote_ref")
|
||||
git push --force-with-lease="refs/heads/$UPDATE_BRANCH:$remote_sha" origin "HEAD:refs/heads/$UPDATE_BRANCH"
|
||||
else
|
||||
git push origin "HEAD:refs/heads/$UPDATE_BRANCH"
|
||||
fi
|
||||
|
||||
body_file="$RUNNER_TEMP/georelay-pr-body.md"
|
||||
{
|
||||
echo "## Automated GeoRelay data proposal"
|
||||
echo
|
||||
echo "- Source: $SOURCE_URL"
|
||||
echo "- Upstream commit: $SOURCE_COMMIT"
|
||||
echo "- Data rows: $DATA_ROWS"
|
||||
echo "- Unique normalized relays: $UNIQUE_RELAYS"
|
||||
echo "- SHA-256: $DATA_SHA256"
|
||||
echo
|
||||
echo "The candidate passed strict UTF-8, schema, size, row-count, secure-host, coordinate, duplicate-conflict, and baseline-delta validation."
|
||||
echo
|
||||
echo "This PR is intentionally not auto-merged. Review the relay additions/removals before merging."
|
||||
} > "$body_file"
|
||||
|
||||
existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty')
|
||||
pr_error="$RUNNER_TEMP/georelay-pr-error.txt"
|
||||
pr_url=""
|
||||
if [[ -n "$existing_pr" ]]; then
|
||||
if gh pr edit "$existing_pr" --repo "$GITHUB_REPOSITORY" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"; then
|
||||
pr_url=$(gh pr view "$existing_pr" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||
fi
|
||||
else
|
||||
if created_pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$UPDATE_BRANCH" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"); then
|
||||
pr_url="$created_pr_url"
|
||||
fi
|
||||
fi
|
||||
|
||||
tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number")
|
||||
tracking_issues=()
|
||||
if [[ -n "$tracking_issue_numbers" ]]; then
|
||||
mapfile -t tracking_issues <<< "$tracking_issue_numbers"
|
||||
fi
|
||||
|
||||
if [[ -n "$pr_url" ]]; then
|
||||
for issue_number in "${tracking_issues[@]}"; do
|
||||
gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "A pull request is now available at $pr_url; closing this fallback tracking issue."
|
||||
done
|
||||
echo "Published GeoRelay review PR: $pr_url" >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "::warning::GITHUB_TOKEN could not create or update the GeoRelay pull request; publishing the issues-write fallback."
|
||||
if [[ -s "$pr_error" ]]; then
|
||||
cat "$pr_error" >&2
|
||||
fi
|
||||
|
||||
compare_url="https://github.com/${GITHUB_REPOSITORY}/compare/main...${UPDATE_BRANCH}?expand=1"
|
||||
issue_body_file="$RUNNER_TEMP/georelay-tracking-issue-body.md"
|
||||
{
|
||||
echo "## Validated GeoRelay update awaiting review"
|
||||
echo
|
||||
echo "The automation branch was updated, but this workflow token could not create or update the pull request. Use the compare link below to create it manually."
|
||||
echo
|
||||
echo "- Compare and create PR: $compare_url"
|
||||
echo "- Automation branch: $UPDATE_BRANCH"
|
||||
echo "- Source: $SOURCE_URL"
|
||||
echo "- Upstream commit: $SOURCE_COMMIT"
|
||||
echo "- Data rows: $DATA_ROWS"
|
||||
echo "- Unique normalized relays: $UNIQUE_RELAYS"
|
||||
echo "- SHA-256: $DATA_SHA256"
|
||||
echo
|
||||
echo "The snapshot passed the repository's strict validator before the branch was pushed."
|
||||
} > "$issue_body_file"
|
||||
|
||||
if (( ${#tracking_issues[@]} > 0 )); then
|
||||
primary_issue="${tracking_issues[0]}"
|
||||
gh issue edit "$primary_issue" --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file"
|
||||
issue_url=$(gh issue view "$primary_issue" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||
for duplicate_issue in "${tracking_issues[@]:1}"; do
|
||||
gh issue close "$duplicate_issue" --repo "$GITHUB_REPOSITORY" --comment "Closing duplicate GeoRelay automation tracking issue; #$primary_issue is canonical."
|
||||
done
|
||||
else
|
||||
issue_url=$(gh issue create --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file")
|
||||
fi
|
||||
|
||||
# Do not claim success until the fallback issue was confirmed.
|
||||
[[ -n "$issue_url" ]]
|
||||
echo "Published GeoRelay tracking issue fallback: $issue_url" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Clean obsolete automation review state
|
||||
if: steps.changes.outputs.changed == 'false'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh auth setup-git
|
||||
|
||||
existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty')
|
||||
if [[ -n "$existing_pr" ]]; then
|
||||
gh pr close "$existing_pr" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation proposal."
|
||||
echo "Closed obsolete PR #$existing_pr." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number")
|
||||
if [[ -n "$tracking_issue_numbers" ]]; then
|
||||
while IFS= read -r issue_number; do
|
||||
gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation tracker."
|
||||
echo "Closed obsolete tracking issue #$issue_number." >> "$GITHUB_STEP_SUMMARY"
|
||||
done <<< "$tracking_issue_numbers"
|
||||
fi
|
||||
|
||||
if git ls-remote --exit-code --heads origin "refs/heads/$UPDATE_BRANCH" > /dev/null; then
|
||||
git push origin --delete "$UPDATE_BRANCH"
|
||||
echo "Deleted obsolete automation branch $UPDATE_BRANCH." >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
ls_remote_status=$?
|
||||
if (( ls_remote_status != 2 )); then
|
||||
echo "::error::Could not inspect the obsolete automation branch"
|
||||
exit "$ls_remote_status"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -32,6 +32,23 @@ struct GeoRelayDirectoryDependencies {
|
||||
var retrySleep: (TimeInterval) async -> Void
|
||||
var activeNotificationName: Notification.Name?
|
||||
var autoStart: Bool
|
||||
var validationPolicy: GeoRelayDirectoryValidationPolicy
|
||||
}
|
||||
|
||||
struct GeoRelayDirectoryValidationPolicy: Sendable {
|
||||
let maximumBytes: Int
|
||||
let maximumRows: Int
|
||||
let maximumEntries: Int
|
||||
let minimumRemoteEntries: Int
|
||||
let minimumRetainedFraction: Double
|
||||
|
||||
static let live = GeoRelayDirectoryValidationPolicy(
|
||||
maximumBytes: 512 * 1024,
|
||||
maximumRows: 5_000,
|
||||
maximumEntries: 5_000,
|
||||
minimumRemoteEntries: 50,
|
||||
minimumRetainedFraction: 0.5
|
||||
)
|
||||
}
|
||||
|
||||
private extension GeoRelayDirectoryDependencies {
|
||||
@@ -44,12 +61,16 @@ private extension GeoRelayDirectoryDependencies {
|
||||
#else
|
||||
let activeNotificationName: Notification.Name? = nil
|
||||
#endif
|
||||
let validationPolicy = GeoRelayDirectoryValidationPolicy.live
|
||||
|
||||
return Self(
|
||||
userDefaults: .standard,
|
||||
notificationCenter: .default,
|
||||
now: Date.init,
|
||||
remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv")!,
|
||||
// Runtime refreshes only from bitchat's reviewed copy. Upstream
|
||||
// georelays/main is imported by a validator-backed pull request,
|
||||
// so an upstream mutation cannot immediately retarget clients.
|
||||
remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/bitchat/refs/heads/main/relays/online_relays_gps.csv")!,
|
||||
fetchInterval: TransportConfig.geoRelayFetchIntervalSeconds,
|
||||
refreshCheckInterval: TransportConfig.geoRelayRefreshCheckIntervalSeconds,
|
||||
retryInitialSeconds: TransportConfig.geoRelayRetryInitialSeconds,
|
||||
@@ -58,7 +79,27 @@ private extension GeoRelayDirectoryDependencies {
|
||||
makeFetchData: {
|
||||
let session = TorURLSession.shared.session
|
||||
return { request in
|
||||
let (data, _) = try await session.data(for: request)
|
||||
let (bytes, response) = try await session.bytes(for: request)
|
||||
guard let response = response as? HTTPURLResponse,
|
||||
(200...299).contains(response.statusCode),
|
||||
response.url == request.url else {
|
||||
throw URLError(.badServerResponse)
|
||||
}
|
||||
|
||||
let maximumBytes = validationPolicy.maximumBytes
|
||||
guard response.expectedContentLength <= Int64(maximumBytes) else {
|
||||
throw URLError(.dataLengthExceedsMaximum)
|
||||
}
|
||||
var data = Data()
|
||||
if response.expectedContentLength > 0 {
|
||||
data.reserveCapacity(Int(response.expectedContentLength))
|
||||
}
|
||||
for try await byte in bytes {
|
||||
guard data.count < maximumBytes else {
|
||||
throw URLError(.dataLengthExceedsMaximum)
|
||||
}
|
||||
data.append(byte)
|
||||
}
|
||||
return data
|
||||
}
|
||||
},
|
||||
@@ -76,7 +117,11 @@ private extension GeoRelayDirectoryDependencies {
|
||||
)
|
||||
let dir = base.appendingPathComponent("bitchat", isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
||||
return dir.appendingPathComponent("georelays_cache.csv")
|
||||
// v2 ignores caches populated from the old direct-upstream
|
||||
// trust path and subjects every load to strict validation.
|
||||
let legacyCache = dir.appendingPathComponent("georelays_cache.csv")
|
||||
try? FileManager.default.removeItem(at: legacyCache)
|
||||
return dir.appendingPathComponent("georelays_cache_v2.csv")
|
||||
} catch {
|
||||
return nil
|
||||
}
|
||||
@@ -94,7 +139,8 @@ private extension GeoRelayDirectoryDependencies {
|
||||
try? await Task.sleep(nanoseconds: nanoseconds)
|
||||
},
|
||||
activeNotificationName: activeNotificationName,
|
||||
autoStart: true
|
||||
autoStart: true,
|
||||
validationPolicy: validationPolicy
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -125,7 +171,7 @@ final class GeoRelayDirectory {
|
||||
}
|
||||
|
||||
private enum DetachedFetchOutcome: Sendable {
|
||||
case success(entries: [Entry], csv: String)
|
||||
case success(entries: [Entry], csv: Data)
|
||||
case torNotReady
|
||||
case invalidData
|
||||
case network(String)
|
||||
@@ -212,6 +258,8 @@ final class GeoRelayDirectory {
|
||||
)
|
||||
let awaitTorReady = dependencies.awaitTorReady
|
||||
let fetchData = dependencies.makeFetchData()
|
||||
let validationPolicy = dependencies.validationPolicy
|
||||
let baselineEntries = Set(entries)
|
||||
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
@@ -219,7 +267,9 @@ final class GeoRelayDirectory {
|
||||
let outcome = await Self.fetchRemoteOutcome(
|
||||
request: request,
|
||||
awaitTorReady: awaitTorReady,
|
||||
fetchData: fetchData
|
||||
fetchData: fetchData,
|
||||
validationPolicy: validationPolicy,
|
||||
baselineEntries: baselineEntries
|
||||
)
|
||||
|
||||
switch outcome {
|
||||
@@ -238,7 +288,9 @@ final class GeoRelayDirectory {
|
||||
nonisolated private static func fetchRemoteOutcome(
|
||||
request: URLRequest,
|
||||
awaitTorReady: @escaping @Sendable () async -> Bool,
|
||||
fetchData: @escaping @Sendable (URLRequest) async throws -> Data
|
||||
fetchData: @escaping @Sendable (URLRequest) async throws -> Data,
|
||||
validationPolicy: GeoRelayDirectoryValidationPolicy,
|
||||
baselineEntries: Set<Entry>
|
||||
) async -> DetachedFetchOutcome {
|
||||
await Task.detached(priority: .utility) {
|
||||
let ready = await awaitTorReady()
|
||||
@@ -246,16 +298,16 @@ final class GeoRelayDirectory {
|
||||
|
||||
do {
|
||||
let data = try await fetchData(request)
|
||||
guard let text = String(data: data, encoding: .utf8) else {
|
||||
guard let parsed = Self.validatedEntries(
|
||||
from: data,
|
||||
policy: validationPolicy,
|
||||
minimumEntries: validationPolicy.minimumRemoteEntries,
|
||||
baselineEntries: baselineEntries
|
||||
) else {
|
||||
return .invalidData
|
||||
}
|
||||
|
||||
let parsed = Self.parseCSV(text)
|
||||
guard !parsed.isEmpty else {
|
||||
return .invalidData
|
||||
}
|
||||
|
||||
return .success(entries: parsed, csv: text)
|
||||
return .success(entries: parsed, csv: data)
|
||||
} catch {
|
||||
return .network(error.localizedDescription)
|
||||
}
|
||||
@@ -269,7 +321,7 @@ final class GeoRelayDirectory {
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func handleFetchSuccess(entries parsed: [Entry], csv: String) {
|
||||
private func handleFetchSuccess(entries parsed: [Entry], csv: Data) {
|
||||
entries = parsed
|
||||
persistCache(csv)
|
||||
dependencies.userDefaults.set(dependencies.now(), forKey: lastFetchKey)
|
||||
@@ -321,9 +373,8 @@ final class GeoRelayDirectory {
|
||||
cleanupState.retryTask = nil
|
||||
}
|
||||
|
||||
private func persistCache(_ text: String) {
|
||||
private func persistCache(_ data: Data) {
|
||||
guard let url = dependencies.cacheURL() else { return }
|
||||
guard let data = text.data(using: .utf8) else { return }
|
||||
do {
|
||||
try dependencies.writeData(data, url)
|
||||
} catch {
|
||||
@@ -336,9 +387,12 @@ final class GeoRelayDirectory {
|
||||
// Prefer cached file if present
|
||||
if let cache = dependencies.cacheURL(),
|
||||
let data = dependencies.readData(cache),
|
||||
let text = String(data: data, encoding: .utf8) {
|
||||
let arr = Self.parseCSV(text)
|
||||
if !arr.isEmpty { return arr }
|
||||
let entries = Self.validatedEntries(
|
||||
from: data,
|
||||
policy: dependencies.validationPolicy,
|
||||
minimumEntries: 1
|
||||
) {
|
||||
return entries
|
||||
}
|
||||
|
||||
// Try bundled resource(s)
|
||||
@@ -346,36 +400,157 @@ final class GeoRelayDirectory {
|
||||
|
||||
for url in bundleCandidates {
|
||||
if let data = dependencies.readData(url),
|
||||
let text = String(data: data, encoding: .utf8) {
|
||||
let arr = Self.parseCSV(text)
|
||||
if !arr.isEmpty { return arr }
|
||||
let entries = Self.validatedEntries(
|
||||
from: data,
|
||||
policy: dependencies.validationPolicy,
|
||||
minimumEntries: 1
|
||||
) {
|
||||
return entries
|
||||
}
|
||||
}
|
||||
|
||||
// Try filesystem path (development/test)
|
||||
if let cwd = dependencies.currentDirectoryPath(),
|
||||
let data = dependencies.readData(URL(fileURLWithPath: cwd).appendingPathComponent("relays/online_relays_gps.csv")),
|
||||
let text = String(data: data, encoding: .utf8) {
|
||||
return Self.parseCSV(text)
|
||||
let entries = Self.validatedEntries(
|
||||
from: data,
|
||||
policy: dependencies.validationPolicy,
|
||||
minimumEntries: 1
|
||||
) {
|
||||
return entries
|
||||
}
|
||||
|
||||
SecureLogger.warning("GeoRelayDirectory: no local CSV found; entries empty", category: .session)
|
||||
return []
|
||||
}
|
||||
|
||||
nonisolated static func parseCSV(_ text: String) -> [Entry] {
|
||||
var result: Set<Entry> = []
|
||||
let lines = text.split(whereSeparator: { $0.isNewline })
|
||||
for (idx, raw) in lines.enumerated() {
|
||||
guard let line = raw.trimmedOrNilIfEmpty else { continue }
|
||||
if idx == 0 && line.lowercased().contains("relay url") { continue }
|
||||
let parts = line.split(separator: ",").map { $0.trimmed }
|
||||
guard parts.count >= 3 else { continue }
|
||||
guard let host = NostrRelayURL.directoryAddress(parts[0]) else { continue }
|
||||
guard let lat = Double(parts[1]), let lon = Double(parts[2]) else { continue }
|
||||
result.insert(Entry(host: host, lat: lat, lon: lon))
|
||||
/// Parses the fixed three-column format as an all-or-nothing trust unit.
|
||||
/// One malformed or conflicting row rejects the complete dataset rather
|
||||
/// than silently shrinking or partially replacing the current directory.
|
||||
nonisolated static func validatedEntries(
|
||||
from data: Data,
|
||||
policy: GeoRelayDirectoryValidationPolicy,
|
||||
minimumEntries: Int,
|
||||
baselineEntries: Set<Entry>? = nil
|
||||
) -> [Entry]? {
|
||||
guard !data.isEmpty, data.count <= policy.maximumBytes,
|
||||
let text = String(data: data, encoding: .utf8),
|
||||
!text.hasPrefix("\u{feff}") else {
|
||||
return nil
|
||||
}
|
||||
return Array(result)
|
||||
|
||||
let lines = text.split(whereSeparator: { $0.isNewline })
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.filter { !$0.isEmpty }
|
||||
guard let header = lines.first,
|
||||
lines.count - 1 <= policy.maximumRows else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let headerParts = header
|
||||
.split(separator: ",", omittingEmptySubsequences: false)
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }
|
||||
let supportedHeaders = [
|
||||
["relay url", "latitude", "longitude"],
|
||||
["relay url", "lat", "lon"]
|
||||
]
|
||||
guard supportedHeaders.contains(headerParts) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entriesByHost: [String: Entry] = [:]
|
||||
for line in lines.dropFirst() {
|
||||
let parts = line
|
||||
.split(separator: ",", omittingEmptySubsequences: false)
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
guard parts.count == 3,
|
||||
let host = validatedDirectoryAddress(parts[0]),
|
||||
let latitude = Double(parts[1]), latitude.isFinite,
|
||||
(-90.0...90.0).contains(latitude),
|
||||
let longitude = Double(parts[2]), longitude.isFinite,
|
||||
(-180.0...180.0).contains(longitude) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let entry = Entry(host: host, lat: latitude, lon: longitude)
|
||||
if let existing = entriesByHost[host], existing != entry {
|
||||
// One endpoint cannot truthfully occupy two coordinates. Do
|
||||
// not let row ordering choose which location clients trust.
|
||||
return nil
|
||||
}
|
||||
entriesByHost[host] = entry
|
||||
guard entriesByHost.count <= policy.maximumEntries else { return nil }
|
||||
}
|
||||
|
||||
let parsedEntries = Set(entriesByHost.values)
|
||||
guard parsedEntries.count >= minimumEntries else { return nil }
|
||||
|
||||
if let baselineEntries {
|
||||
guard (0...1).contains(policy.minimumRetainedFraction) else { return nil }
|
||||
let requiredOverlap = Int(
|
||||
ceil(Double(baselineEntries.count) * policy.minimumRetainedFraction)
|
||||
)
|
||||
guard parsedEntries.intersection(baselineEntries).count >= requiredOverlap else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return parsedEntries.sorted {
|
||||
($0.host, $0.lat, $0.lon) < ($1.host, $1.lat, $1.lon)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated private static func validatedDirectoryAddress(_ rawValue: String) -> String? {
|
||||
let value = rawValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !value.isEmpty,
|
||||
value.unicodeScalars.allSatisfy({
|
||||
$0.isASCII && !CharacterSet.controlCharacters.contains($0)
|
||||
}) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let candidate = value.contains("://") ? value : "wss://\(value)"
|
||||
guard let components = URLComponents(string: candidate),
|
||||
let scheme = components.scheme?.lowercased(),
|
||||
scheme == "wss" || scheme == "https",
|
||||
components.user == nil,
|
||||
components.password == nil,
|
||||
components.query == nil,
|
||||
components.fragment == nil,
|
||||
components.path.isEmpty || components.path == "/",
|
||||
let rawHost = components.host else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let host = rawHost.lowercased()
|
||||
guard !host.isEmpty, host.count <= 253,
|
||||
host.unicodeScalars.allSatisfy({ $0.isASCII }),
|
||||
!host.hasSuffix("."),
|
||||
host != "localhost",
|
||||
!host.hasSuffix(".localhost"),
|
||||
!host.hasSuffix(".local"),
|
||||
!host.hasSuffix(".internal") else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let labels = host.split(separator: ".", omittingEmptySubsequences: false)
|
||||
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyz0123456789-")
|
||||
guard labels.count >= 2,
|
||||
!labels.allSatisfy({ $0.allSatisfy(\.isNumber) }),
|
||||
labels.allSatisfy({ label in
|
||||
(1...63).contains(label.count) &&
|
||||
label.first != "-" &&
|
||||
label.last != "-" &&
|
||||
label.unicodeScalars.allSatisfy { allowed.contains($0) }
|
||||
}) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
if let port = components.port {
|
||||
guard (1...65_535).contains(port) else { return nil }
|
||||
if port != 443 { return "\(host):\(port)" }
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// MARK: - Observers & Timers
|
||||
|
||||
@@ -39,13 +39,4 @@ enum NostrRelayURL {
|
||||
|
||||
return components.string
|
||||
}
|
||||
|
||||
static func directoryAddress(_ rawValue: String) -> String? {
|
||||
guard var normalized = normalized(rawValue, defaultScheme: "wss") else { return nil }
|
||||
for prefix in ["wss://", "ws://"] where normalized.hasPrefix(prefix) {
|
||||
normalized.removeFirst(prefix.count)
|
||||
break
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,19 +5,25 @@ import XCTest
|
||||
|
||||
@MainActor
|
||||
final class GeoRelayDirectoryTests: XCTestCase {
|
||||
func test_parseCSV_normalizesRelaySchemesAndDeduplicatesEntries() {
|
||||
private func parse(_ csv: String) -> [GeoRelayDirectory.Entry] {
|
||||
GeoRelayDirectory.validatedEntries(
|
||||
from: Data(csv.utf8),
|
||||
policy: .live,
|
||||
minimumEntries: 1
|
||||
) ?? []
|
||||
}
|
||||
|
||||
func test_parseCSV_normalizesSecureRelaySchemesAndDeduplicatesEntries() {
|
||||
let csv = """
|
||||
relay url,lat,lon
|
||||
wss://one.example/,10,20
|
||||
https://one.example,10,20
|
||||
wss://one.example:443/,10,20
|
||||
http://two.example/,11,21
|
||||
two.example,11,21
|
||||
wss://two.example:443,11,21
|
||||
invalid row
|
||||
ws://three.example,not-a-lat,22
|
||||
"""
|
||||
|
||||
let parsed = Set(GeoRelayDirectory.parseCSV(csv))
|
||||
let parsed = Set(parse(csv))
|
||||
|
||||
XCTAssertEqual(
|
||||
parsed,
|
||||
@@ -28,6 +34,136 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
||||
)
|
||||
}
|
||||
|
||||
func test_parseCSV_rejectsWholeDatasetWhenAnyRowOrHeaderIsUnsafe() {
|
||||
let invalidCSVs = [
|
||||
"relay,lat,lon\nrelay.example,1,2\n",
|
||||
"relay url,lat,lon\nrelay.example,1\n",
|
||||
"relay url,lat,lon\nhttp://relay.example,1,2\n",
|
||||
"relay url,lat,lon\nwss://user@relay.example,1,2\n",
|
||||
"relay url,lat,lon\nwss://relay.example/path,1,2\n",
|
||||
"relay url,lat,lon\nwss://relay.example?,1,2\n",
|
||||
"relay url,lat,lon\nwss://relay.example#,1,2\n",
|
||||
"relay url,lat,lon\nrelay.example:0,1,2\n",
|
||||
"relay url,lat,lon\nrelay.example:99999,1,2\n",
|
||||
"relay url,lat,lon\nlocalhost,1,2\n",
|
||||
"relay url,lat,lon\nr\u{00e9}lay.example,1,2\n",
|
||||
"relay url,lat,lon\nrelay\u{202e}.example,1,2\n",
|
||||
"relay url,lat,lon\nrelay.example,NaN,2\n",
|
||||
"relay url,lat,lon\nrelay.example,1_0,2\n",
|
||||
"relay url,lat,lon\nrelay.example,\u{0661}\u{0660},2\n",
|
||||
"relay url,lat,lon\nrelay.example,\u{ff11}\u{ff10},2\n",
|
||||
"relay url,lat,lon\nrelay.example,91,2\n",
|
||||
"relay url,lat,lon\nrelay.example,1,181\n",
|
||||
"relay url,lat,lon\nrelay.example,1,2\nrelay.example,3,4\n"
|
||||
]
|
||||
|
||||
for csv in invalidCSVs {
|
||||
XCTAssertTrue(parse(csv).isEmpty, csv)
|
||||
}
|
||||
}
|
||||
|
||||
func test_validatedEntries_enforcesByteRowEntryAndRetentionLimits() {
|
||||
let restrictive = GeoRelayDirectoryValidationPolicy(
|
||||
maximumBytes: 100,
|
||||
maximumRows: 2,
|
||||
maximumEntries: 2,
|
||||
minimumRemoteEntries: 1,
|
||||
minimumRetainedFraction: 0.5
|
||||
)
|
||||
let one = Data("relay url,lat,lon\none.example,1,2\n".utf8)
|
||||
let three = Data("relay url,lat,lon\none.example,1,2\ntwo.example,3,4\nthree.example,5,6\n".utf8)
|
||||
|
||||
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||
from: one,
|
||||
policy: restrictive,
|
||||
minimumEntries: 2
|
||||
))
|
||||
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||
from: Data(repeating: 0x41, count: 101),
|
||||
policy: restrictive,
|
||||
minimumEntries: 1
|
||||
))
|
||||
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||
from: three,
|
||||
policy: restrictive,
|
||||
minimumEntries: 1
|
||||
))
|
||||
}
|
||||
|
||||
func test_validatedEntries_requiresExactBaselineEntryOverlap() throws {
|
||||
let policy = GeoRelayDirectoryValidationPolicy(
|
||||
maximumBytes: 1_000,
|
||||
maximumRows: 10,
|
||||
maximumEntries: 10,
|
||||
minimumRemoteEntries: 1,
|
||||
minimumRetainedFraction: 0.5
|
||||
)
|
||||
let baseline = Set(try XCTUnwrap(GeoRelayDirectory.validatedEntries(
|
||||
from: Data("""
|
||||
relay url,lat,lon
|
||||
one.example,1,1
|
||||
two.example,2,2
|
||||
three.example,3,3
|
||||
""".utf8),
|
||||
policy: policy,
|
||||
minimumEntries: 1
|
||||
)))
|
||||
let disjoint = Data("""
|
||||
relay url,lat,lon
|
||||
four.example,1,1
|
||||
five.example,2,2
|
||||
six.example,3,3
|
||||
""".utf8)
|
||||
let rewrittenCoordinates = Data("""
|
||||
relay url,lat,lon
|
||||
one.example,11,11
|
||||
two.example,12,12
|
||||
three.example,13,13
|
||||
""".utf8)
|
||||
let halfRetained = Data("""
|
||||
relay url,lat,lon
|
||||
wss://one.example:443/,1,1
|
||||
https://two.example/,2,2
|
||||
replacement.example,4,4
|
||||
""".utf8)
|
||||
|
||||
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||
from: disjoint,
|
||||
policy: policy,
|
||||
minimumEntries: 1,
|
||||
baselineEntries: baseline
|
||||
))
|
||||
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||
from: rewrittenCoordinates,
|
||||
policy: policy,
|
||||
minimumEntries: 1,
|
||||
baselineEntries: baseline
|
||||
))
|
||||
XCTAssertNotNil(GeoRelayDirectory.validatedEntries(
|
||||
from: halfRetained,
|
||||
policy: policy,
|
||||
minimumEntries: 1,
|
||||
baselineEntries: baseline
|
||||
))
|
||||
}
|
||||
|
||||
func test_bundledReviewedCSV_passesStrictProductionValidation() throws {
|
||||
let repositoryRoot = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
let data = try Data(
|
||||
contentsOf: repositoryRoot.appendingPathComponent("relays/online_relays_gps.csv")
|
||||
)
|
||||
|
||||
let entries = try XCTUnwrap(GeoRelayDirectory.validatedEntries(
|
||||
from: data,
|
||||
policy: .live,
|
||||
minimumEntries: GeoRelayDirectoryValidationPolicy.live.minimumRemoteEntries
|
||||
))
|
||||
XCTAssertGreaterThan(entries.count, 250)
|
||||
}
|
||||
|
||||
func test_closestRelays_sortsByDistanceForLatLonAndGeohash() {
|
||||
let harness = makeHarness(
|
||||
cacheCSV: """
|
||||
@@ -243,6 +379,53 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
||||
XCTAssertFalse(directory.debugHasRetryTask)
|
||||
}
|
||||
|
||||
func test_prefetchIfNeeded_rejectsSharpValidLookingTruncationBeforeCaching() async {
|
||||
let cached = """
|
||||
relay url,lat,lon
|
||||
old-one.example,1,1
|
||||
old-two.example,2,2
|
||||
old-three.example,3,3
|
||||
"""
|
||||
let truncated = """
|
||||
relay url,lat,lon
|
||||
attacker.example,9,9
|
||||
"""
|
||||
let recovered = """
|
||||
relay url,lat,lon
|
||||
old-one.example,1,1
|
||||
old-two.example,2,2
|
||||
new-three.example,6,6
|
||||
"""
|
||||
let harness = makeHarness(
|
||||
cacheCSV: cached,
|
||||
fetchResults: [
|
||||
.success(Data(truncated.utf8)),
|
||||
.success(Data(recovered.utf8))
|
||||
],
|
||||
validationPolicy: GeoRelayDirectoryValidationPolicy(
|
||||
maximumBytes: 64 * 1024,
|
||||
maximumRows: 1_000,
|
||||
maximumEntries: 1_000,
|
||||
minimumRemoteEntries: 1,
|
||||
minimumRetainedFraction: 0.5
|
||||
)
|
||||
)
|
||||
let directory = GeoRelayDirectory(dependencies: harness.dependencies)
|
||||
|
||||
directory.prefetchIfNeeded()
|
||||
|
||||
let refreshed = await waitUntil {
|
||||
directory.entries.contains(where: { $0.host == "new-three.example" })
|
||||
}
|
||||
XCTAssertTrue(refreshed)
|
||||
XCTAssertFalse(directory.entries.contains(where: { $0.host == "attacker.example" }))
|
||||
let requestCount = await harness.fetcher.recordedRequestCount()
|
||||
let retryDelays = await harness.retryRecorder.recordedDelays()
|
||||
XCTAssertEqual(requestCount, 2)
|
||||
XCTAssertEqual(retryDelays, [5])
|
||||
XCTAssertEqual(harness.fileStore.dataByURL[harness.cacheURL], Data(recovered.utf8))
|
||||
}
|
||||
|
||||
func test_observers_triggerPrefetchesForTorReadyAndAppActivation() async {
|
||||
let activeNotification = Notification.Name("GeoRelayDirectoryTests.didBecomeActive")
|
||||
let harness = makeHarness(
|
||||
@@ -289,7 +472,14 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
||||
fetchFactoryObserver: (@MainActor @Sendable () -> Void)? = nil,
|
||||
fetchObserver: (@Sendable () async -> Void)? = nil,
|
||||
autoStart: Bool = false,
|
||||
activeNotificationName: Notification.Name? = nil
|
||||
activeNotificationName: Notification.Name? = nil,
|
||||
validationPolicy: GeoRelayDirectoryValidationPolicy = GeoRelayDirectoryValidationPolicy(
|
||||
maximumBytes: 64 * 1024,
|
||||
maximumRows: 1_000,
|
||||
maximumEntries: 1_000,
|
||||
minimumRemoteEntries: 1,
|
||||
minimumRetainedFraction: 0
|
||||
)
|
||||
) -> GeoRelayHarness {
|
||||
let userDefaultsSuite = "GeoRelayDirectoryTests.\(UUID().uuidString)"
|
||||
let userDefaults = UserDefaults(suiteName: userDefaultsSuite)!
|
||||
@@ -347,7 +537,8 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
||||
await retryRecorder.record(delay)
|
||||
},
|
||||
activeNotificationName: activeNotificationName,
|
||||
autoStart: autoStart
|
||||
autoStart: autoStart,
|
||||
validationPolicy: validationPolicy
|
||||
)
|
||||
|
||||
return GeoRelayHarness(
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW_PATH = REPOSITORY_ROOT / ".github/workflows/fetch_georelays.yml"
|
||||
|
||||
|
||||
class FetchGeoRelaysWorkflowTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_write_capable_checkout_action_is_immutable(self) -> None:
|
||||
checkout = re.search(r"uses: actions/checkout@([0-9a-f]+)", self.workflow)
|
||||
self.assertIsNotNone(checkout)
|
||||
self.assertRegex(checkout.group(1), r"^[0-9a-f]{40}$")
|
||||
self.assertIn("persist-credentials: false", self.workflow)
|
||||
|
||||
def test_pr_failure_has_single_issue_fallback_with_review_metadata(self) -> None:
|
||||
required_fragments = [
|
||||
"issues: write",
|
||||
"TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request",
|
||||
"gh pr create",
|
||||
"gh issue create",
|
||||
"gh issue edit",
|
||||
"compare/main...${UPDATE_BRANCH}?expand=1",
|
||||
"Upstream commit: $SOURCE_COMMIT",
|
||||
"Data rows: $DATA_ROWS",
|
||||
"Unique normalized relays: $UNIQUE_RELAYS",
|
||||
"SHA-256: $DATA_SHA256",
|
||||
'[[ -n "$issue_url" ]]',
|
||||
]
|
||||
for fragment in required_fragments:
|
||||
with self.subTest(fragment=fragment):
|
||||
self.assertIn(fragment, self.workflow)
|
||||
|
||||
confirmed = self.workflow.index('[[ -n "$issue_url" ]]')
|
||||
success_summary = self.workflow.index(
|
||||
"Published GeoRelay tracking issue fallback: $issue_url"
|
||||
)
|
||||
self.assertLess(confirmed, success_summary)
|
||||
|
||||
def test_obsolete_review_state_is_cleaned_without_pushing_main(self) -> None:
|
||||
self.assertIn("gh pr close", self.workflow)
|
||||
self.assertIn("gh issue close", self.workflow)
|
||||
self.assertIn('git push origin --delete "$UPDATE_BRANCH"', self.workflow)
|
||||
self.assertIn('git switch -C "$UPDATE_BRANCH"', self.workflow)
|
||||
self.assertNotIn("git push origin main", self.workflow)
|
||||
self.assertNotIn("git push --force origin main", self.workflow)
|
||||
|
||||
def test_workflow_runs_all_validator_tests(self) -> None:
|
||||
self.assertIn(
|
||||
'python3 -m unittest discover -s scripts/tests -p "test_*.py" -v',
|
||||
self.workflow,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,186 @@
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import validate_georelays as validator
|
||||
|
||||
|
||||
def csv_bytes(rows: list[str]) -> bytes:
|
||||
return ("Relay URL,Latitude,Longitude\n" + "\n".join(rows) + "\n").encode()
|
||||
|
||||
|
||||
class ValidateGeoRelaysTests(unittest.TestCase):
|
||||
def test_validates_and_deduplicates_secure_relay_addresses(self) -> None:
|
||||
data = csv_bytes(
|
||||
[
|
||||
"relay.example.com,10,20",
|
||||
"wss://relay.example.com:443/,10,20",
|
||||
"https://second.example.org,11,21",
|
||||
]
|
||||
)
|
||||
|
||||
summary = validator.validate_bytes(data, minimum_unique_relays=2)
|
||||
|
||||
self.assertEqual(summary.data_rows, 3)
|
||||
self.assertEqual(summary.unique_relays, 2)
|
||||
|
||||
def test_rejects_insecure_or_non_host_relay_urls(self) -> None:
|
||||
bad_addresses = [
|
||||
"http://relay.example.com",
|
||||
"ws://relay.example.com",
|
||||
"wss://user@relay.example.com",
|
||||
"wss://relay.example.com/path",
|
||||
"wss://relay.example.com?",
|
||||
"wss://relay.example.com#",
|
||||
"relay.example.com:0",
|
||||
"relay.example.com:99999",
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"relay_example.com",
|
||||
"relay\u202e.example.com",
|
||||
]
|
||||
|
||||
for address in bad_addresses:
|
||||
with self.subTest(address=address):
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes([f"{address},10,20"]),
|
||||
minimum_unique_relays=1,
|
||||
)
|
||||
|
||||
def test_rejects_malformed_rows_and_unsafe_coordinates(self) -> None:
|
||||
bad_rows = [
|
||||
"relay.example.com,10",
|
||||
"relay.example.com,NaN,20",
|
||||
"relay.example.com,1_0,20",
|
||||
"relay.example.com,\u0661\u0660,20",
|
||||
"relay.example.com,\uff11\uff10,20",
|
||||
"relay.example.com,91,20",
|
||||
"relay.example.com,10,-181",
|
||||
"relay.example.com,10,20,extra",
|
||||
'"relay.example.com",10,20',
|
||||
]
|
||||
|
||||
for row in bad_rows:
|
||||
with self.subTest(row=row):
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(csv_bytes([row]), minimum_unique_relays=1)
|
||||
|
||||
def test_accepts_ascii_coordinate_forms_supported_by_swift_double(self) -> None:
|
||||
summary = validator.validate_bytes(
|
||||
csv_bytes(
|
||||
[
|
||||
"one.example.com,+1,-.5",
|
||||
"two.example.com,1.e1,2E+1",
|
||||
"three.example.com,01,20.",
|
||||
]
|
||||
),
|
||||
minimum_unique_relays=3,
|
||||
)
|
||||
|
||||
self.assertEqual(summary.unique_relays, 3)
|
||||
|
||||
def test_rejects_conflicts_limits_and_large_baseline_deltas(self) -> None:
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes(["relay.example.com,10,20", "relay.example.com,11,21"]),
|
||||
minimum_unique_relays=1,
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(b"x" * 20, maximum_bytes=10, minimum_unique_relays=1)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes(["one.example.com,1,1", "two.example.com,2,2"]),
|
||||
minimum_unique_relays=3,
|
||||
)
|
||||
|
||||
baseline = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(120)]
|
||||
)
|
||||
shrunken = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(59)]
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_update(shrunken, baseline)
|
||||
|
||||
smaller_baseline = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||
)
|
||||
expanded = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(121)]
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_update(expanded, smaller_baseline)
|
||||
|
||||
def test_update_requires_exact_normalized_baseline_entry_overlap(self) -> None:
|
||||
baseline_rows = [
|
||||
f"relay-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
baseline = csv_bytes(baseline_rows)
|
||||
disjoint = csv_bytes(
|
||||
[
|
||||
f"attacker-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
)
|
||||
rewritten_coordinates = csv_bytes(
|
||||
[
|
||||
f"relay-{index}.example.com,{(index % 80) + 0.5},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
)
|
||||
|
||||
for candidate in (disjoint, rewritten_coordinates):
|
||||
with self.subTest(candidate=candidate[:80]):
|
||||
with self.assertRaisesRegex(
|
||||
validator.ValidationError,
|
||||
"exact relay-coordinate entries",
|
||||
):
|
||||
validator.validate_update(candidate, baseline)
|
||||
|
||||
half_retained = csv_bytes(
|
||||
[
|
||||
f"wss://relay-{index}.example.com:443/,{index % 80},{index % 170}"
|
||||
for index in range(30)
|
||||
]
|
||||
+ [
|
||||
f"replacement-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(30)
|
||||
]
|
||||
)
|
||||
summary = validator.validate_update(half_retained, baseline)
|
||||
self.assertEqual(summary.unique_relays, 60)
|
||||
|
||||
def test_cli_copies_only_validated_data_and_emits_review_metadata(self) -> None:
|
||||
rows = [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||
data = csv_bytes(rows)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
candidate = root / "candidate.csv"
|
||||
baseline = root / "baseline.csv"
|
||||
output = root / "output.csv"
|
||||
github_output = root / "github-output.txt"
|
||||
candidate.write_bytes(data)
|
||||
baseline.write_bytes(data)
|
||||
|
||||
result = validator.main(
|
||||
[
|
||||
"--input", str(candidate),
|
||||
"--baseline", str(baseline),
|
||||
"--output", str(output),
|
||||
"--github-output", str(github_output),
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertEqual(output.read_bytes(), data)
|
||||
metadata = github_output.read_text()
|
||||
self.assertIn("unique_relays=60", metadata)
|
||||
self.assertIn("sha256=", metadata)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict validator for the reviewed georelay CSV update workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unicodedata
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
MAX_BYTES = 512 * 1024
|
||||
MAX_ROWS = 5_000
|
||||
MAX_UNIQUE_RELAYS = 5_000
|
||||
MIN_UNIQUE_RELAYS = 50
|
||||
MIN_BASELINE_FRACTION = 0.5
|
||||
MAX_BASELINE_MULTIPLIER = 2.0
|
||||
EXPECTED_HEADER = ("relay url", "latitude", "longitude")
|
||||
ASCII_DECIMAL_PATTERN = re.compile(
|
||||
r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z"
|
||||
)
|
||||
|
||||
|
||||
class ValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidationSummary:
|
||||
data_rows: int
|
||||
unique_relays: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ValidatedDataset:
|
||||
summary: ValidationSummary
|
||||
entries: frozenset[tuple[str, float, float]]
|
||||
|
||||
|
||||
def _has_disallowed_control(value: str) -> bool:
|
||||
return any(
|
||||
unicodedata.category(character) in {"Cc", "Cf"}
|
||||
and character not in {"\r", "\n", "\t"}
|
||||
for character in value
|
||||
)
|
||||
|
||||
|
||||
def normalize_relay_address(raw_value: str) -> str:
|
||||
value = raw_value.strip()
|
||||
if not value or _has_disallowed_control(value):
|
||||
raise ValidationError("relay address is empty or contains control characters")
|
||||
# urlsplit cannot distinguish an absent query/fragment from an explicitly
|
||||
# empty one. Reject the delimiters themselves so this validator matches
|
||||
# URLComponents in the client and reviewed data cannot fail closed there.
|
||||
if "?" in value or "#" in value:
|
||||
raise ValidationError(f"relay query or fragment is not allowed: {value}")
|
||||
|
||||
candidate = value if "://" in value else f"wss://{value}"
|
||||
try:
|
||||
parsed = urlsplit(candidate)
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValidationError(f"invalid relay URL: {value}") from error
|
||||
|
||||
if parsed.scheme.lower() not in {"wss", "https"}:
|
||||
raise ValidationError(f"relay must use wss/https or a bare hostname: {value}")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValidationError(f"relay credentials are not allowed: {value}")
|
||||
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||
raise ValidationError(f"relay path, query, or fragment is not allowed: {value}")
|
||||
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host or len(host) > 253 or not host.isascii():
|
||||
raise ValidationError(f"relay hostname is missing or non-ASCII: {value}")
|
||||
if host.endswith(".") or host == "localhost" or host.endswith((".localhost", ".local", ".internal")):
|
||||
raise ValidationError(f"local or absolute relay hostname is not allowed: {value}")
|
||||
|
||||
labels = host.split(".")
|
||||
if len(labels) < 2 or all(label.isdigit() for label in labels):
|
||||
raise ValidationError(f"relay must use a public DNS hostname: {value}")
|
||||
for label in labels:
|
||||
if not 1 <= len(label) <= 63:
|
||||
raise ValidationError(f"invalid DNS label length: {value}")
|
||||
if label[0] == "-" or label[-1] == "-":
|
||||
raise ValidationError(f"DNS labels cannot start or end with '-': {value}")
|
||||
if any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label):
|
||||
raise ValidationError(f"invalid DNS hostname character: {value}")
|
||||
|
||||
if port is not None and not 1 <= port <= 65_535:
|
||||
raise ValidationError(f"invalid relay port: {value}")
|
||||
if port in {None, 443}:
|
||||
return host
|
||||
return f"{host}:{port}"
|
||||
|
||||
|
||||
def _validated_dataset(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> _ValidatedDataset:
|
||||
if not data or len(data) > maximum_bytes:
|
||||
raise ValidationError(f"CSV must contain 1..{maximum_bytes} bytes")
|
||||
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValidationError("CSV is not valid UTF-8") from error
|
||||
if text.startswith("\ufeff"):
|
||||
raise ValidationError("UTF-8 BOM is not allowed")
|
||||
if _has_disallowed_control(text):
|
||||
raise ValidationError("CSV contains disallowed control characters")
|
||||
# Runtime intentionally implements the fixed three-field schema without
|
||||
# general CSV quoting. Reject quoted variants here so reviewed workflow
|
||||
# output and client-side validation cannot disagree.
|
||||
if '"' in text:
|
||||
raise ValidationError("quoted CSV fields are not allowed")
|
||||
|
||||
reader = csv.reader(io.StringIO(text, newline=""), strict=True)
|
||||
try:
|
||||
header = next(reader)
|
||||
except (StopIteration, csv.Error) as error:
|
||||
raise ValidationError("CSV header is missing") from error
|
||||
normalized_header = tuple(field.strip().lower() for field in header)
|
||||
if normalized_header != EXPECTED_HEADER:
|
||||
raise ValidationError(f"unexpected CSV header: {header!r}")
|
||||
|
||||
data_rows = 0
|
||||
relays: dict[str, tuple[float, float]] = {}
|
||||
try:
|
||||
for row in reader:
|
||||
if not row or all(not field.strip() for field in row):
|
||||
continue
|
||||
data_rows += 1
|
||||
if data_rows > maximum_rows:
|
||||
raise ValidationError(f"CSV exceeds {maximum_rows} data rows")
|
||||
if len(row) != 3:
|
||||
raise ValidationError(f"row {reader.line_num} must contain exactly 3 columns")
|
||||
|
||||
address = normalize_relay_address(row[0])
|
||||
latitude_text = row[1].strip()
|
||||
longitude_text = row[2].strip()
|
||||
if not ASCII_DECIMAL_PATTERN.fullmatch(latitude_text) or not ASCII_DECIMAL_PATTERN.fullmatch(longitude_text):
|
||||
raise ValidationError(
|
||||
f"row {reader.line_num} coordinates must be ASCII decimal numbers"
|
||||
)
|
||||
latitude = float(latitude_text)
|
||||
longitude = float(longitude_text)
|
||||
if not math.isfinite(latitude) or not -90 <= latitude <= 90:
|
||||
raise ValidationError(f"row {reader.line_num} latitude is out of range")
|
||||
if not math.isfinite(longitude) or not -180 <= longitude <= 180:
|
||||
raise ValidationError(f"row {reader.line_num} longitude is out of range")
|
||||
|
||||
coordinates = (latitude, longitude)
|
||||
previous = relays.get(address)
|
||||
if previous is not None and previous != coordinates:
|
||||
raise ValidationError(f"relay {address} has conflicting coordinates")
|
||||
relays[address] = coordinates
|
||||
if len(relays) > maximum_unique_relays:
|
||||
raise ValidationError(f"CSV exceeds {maximum_unique_relays} unique relays")
|
||||
except csv.Error as error:
|
||||
raise ValidationError(f"malformed CSV near line {reader.line_num}") from error
|
||||
|
||||
if len(relays) < minimum_unique_relays:
|
||||
raise ValidationError(
|
||||
f"CSV has {len(relays)} unique relays; minimum is {minimum_unique_relays}"
|
||||
)
|
||||
|
||||
return _ValidatedDataset(
|
||||
summary=ValidationSummary(
|
||||
data_rows=data_rows,
|
||||
unique_relays=len(relays),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
),
|
||||
entries=frozenset(
|
||||
(address, coordinates[0], coordinates[1])
|
||||
for address, coordinates in relays.items()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def validate_bytes(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> ValidationSummary:
|
||||
return _validated_dataset(
|
||||
data,
|
||||
minimum_unique_relays=minimum_unique_relays,
|
||||
maximum_bytes=maximum_bytes,
|
||||
maximum_rows=maximum_rows,
|
||||
maximum_unique_relays=maximum_unique_relays,
|
||||
).summary
|
||||
|
||||
|
||||
def validate_update(candidate: bytes, baseline: bytes) -> ValidationSummary:
|
||||
baseline_dataset = _validated_dataset(baseline, minimum_unique_relays=1)
|
||||
candidate_dataset = _validated_dataset(candidate)
|
||||
baseline_summary = baseline_dataset.summary
|
||||
candidate_summary = candidate_dataset.summary
|
||||
|
||||
minimum_from_baseline = math.ceil(
|
||||
baseline_summary.unique_relays * MIN_BASELINE_FRACTION
|
||||
)
|
||||
maximum_from_baseline = math.floor(
|
||||
baseline_summary.unique_relays * MAX_BASELINE_MULTIPLIER
|
||||
)
|
||||
if candidate_summary.unique_relays < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate loses more than half of the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} < {minimum_from_baseline})"
|
||||
)
|
||||
if candidate_summary.unique_relays > maximum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate more than doubles the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} > {maximum_from_baseline})"
|
||||
)
|
||||
|
||||
retained_entries = len(baseline_dataset.entries & candidate_dataset.entries)
|
||||
if retained_entries < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate retains fewer than half of the baseline's exact relay-coordinate entries "
|
||||
f"({retained_entries} < {minimum_from_baseline})"
|
||||
)
|
||||
return candidate_summary
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", required=True, type=Path)
|
||||
parser.add_argument("--baseline", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--github-output", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
candidate = args.input.read_bytes()
|
||||
baseline = args.baseline.read_bytes()
|
||||
summary = validate_update(candidate, baseline)
|
||||
args.output.write_bytes(candidate)
|
||||
if args.github_output is not None:
|
||||
with args.github_output.open("a", encoding="utf-8") as output:
|
||||
output.write(f"data_rows={summary.data_rows}\n")
|
||||
output.write(f"unique_relays={summary.unique_relays}\n")
|
||||
output.write(f"sha256={summary.sha256}\n")
|
||||
except (OSError, ValidationError) as error:
|
||||
print(f"georelay validation failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"validated {summary.unique_relays} unique relays across "
|
||||
f"{summary.data_rows} rows (sha256 {summary.sha256})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user