mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-24 22:25:18 +00:00
Route georelay updates through review
This commit is contained in:
@@ -1,42 +1,228 @@
|
|||||||
name: Fetch GeoRelays Data
|
name: Propose GeoRelay Data Update
|
||||||
|
|
||||||
on:
|
on:
|
||||||
schedule:
|
schedule:
|
||||||
- cron: '0 6 * * 0'
|
- cron: "0 6 * * 0"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
|
# Default to read-only. The publishing job receives only the scopes required
|
||||||
|
# to push its branch and publish either a PR or a tracking issue.
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: read
|
||||||
pull-requests: write
|
|
||||||
|
concurrency:
|
||||||
|
group: georelay-data-update
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
SOURCE_REPOSITORY: https://github.com/permissionlesstech/georelays.git
|
||||||
|
UPDATE_BRANCH: automation/georelay-data
|
||||||
|
TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
update-relay-data:
|
propose-relay-data:
|
||||||
|
name: Validate and propose relay data
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
|
issues: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout reviewed base
|
||||||
uses: actions/checkout@v4
|
# Pinned actions/checkout v5 so a mutable action tag cannot change the
|
||||||
|
# code that receives this job's write-capable token.
|
||||||
|
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
ref: main
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
# Do not expose the write token to fetch/validation subprocesses.
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Fetch GeoRelays
|
- name: Test GeoRelay validator
|
||||||
run: |
|
run: |
|
||||||
wget -q https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv
|
set -euo pipefail
|
||||||
mv nostr_relays.csv ./relays/online_relays_gps.csv
|
python3 -m unittest discover -s scripts/tests -p "test_*.py" -v
|
||||||
|
|
||||||
- name: Check for changes
|
- name: Fetch candidate over pinned HTTPS policy
|
||||||
id: git-check
|
id: upstream
|
||||||
run: |
|
run: |
|
||||||
git diff --exit-code || echo "changes=true" >> $GITHUB_OUTPUT
|
set -euo pipefail
|
||||||
|
source_commit=$(git ls-remote --refs "$SOURCE_REPOSITORY" refs/heads/main | awk 'NR == 1 { print $1 }')
|
||||||
- name: Commit and push changes
|
if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||||
if: steps.git-check.outputs.changes == 'true'
|
echo "::error::Could not resolve an immutable upstream commit"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
source_url="https://raw.githubusercontent.com/permissionlesstech/georelays/$source_commit/nostr_relays.csv"
|
||||||
|
effective_url=$(curl --fail --show-error --silent --location --proto "=https" --proto-redir "=https" --tlsv1.2 --max-time 60 --retry 3 --retry-all-errors --output "$RUNNER_TEMP/georelays-candidate.csv" --write-out "%{url_effective}" "$source_url")
|
||||||
|
if [[ "$effective_url" != "$source_url" ]]; then
|
||||||
|
echo "::error::Unexpected GeoRelay redirect target: $effective_url"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "source_commit=$source_commit" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "source_url=$source_url" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Validate candidate against reviewed baseline
|
||||||
|
id: validation
|
||||||
run: |
|
run: |
|
||||||
git config --local user.email "action@github.com"
|
set -euo pipefail
|
||||||
git config --local user.name "GitHub Action"
|
python3 scripts/validate_georelays.py --input "$RUNNER_TEMP/georelays-candidate.csv" --baseline relays/online_relays_gps.csv --output relays/online_relays_gps.csv --github-output "$GITHUB_OUTPUT"
|
||||||
git add relays/online_relays_gps.csv
|
|
||||||
git commit -m "Automated update of relay data - $(date -u)"
|
- name: Check for a reviewed-file change
|
||||||
git push
|
id: changes
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if git diff --quiet -- relays/online_relays_gps.csv; then
|
||||||
|
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "Upstream GeoRelay data already matches main." >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
else
|
||||||
|
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||||
|
git diff --stat -- relays/online_relays_gps.csv
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Push automation branch and publish review request
|
||||||
|
if: steps.changes.outputs.changed == 'true'
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
SOURCE_COMMIT: ${{ steps.upstream.outputs.source_commit }}
|
||||||
|
SOURCE_URL: ${{ steps.upstream.outputs.source_url }}
|
||||||
|
DATA_ROWS: ${{ steps.validation.outputs.data_rows }}
|
||||||
|
UNIQUE_RELAYS: ${{ steps.validation.outputs.unique_relays }}
|
||||||
|
DATA_SHA256: ${{ steps.validation.outputs.sha256 }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Scope credential exposure to this final publishing step.
|
||||||
|
gh auth setup-git
|
||||||
|
git config user.name "github-actions[bot]"
|
||||||
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||||
|
|
||||||
|
git switch -C "$UPDATE_BRANCH"
|
||||||
|
git add -- relays/online_relays_gps.csv
|
||||||
|
git diff --cached --quiet && {
|
||||||
|
echo "::error::Expected a staged GeoRelay data change"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git commit -m "Update reviewed georelay directory" -m "Upstream-commit: $SOURCE_COMMIT"
|
||||||
|
|
||||||
|
remote_ref="refs/remotes/origin/$UPDATE_BRANCH"
|
||||||
|
if git fetch --no-tags origin "+refs/heads/$UPDATE_BRANCH:$remote_ref" 2>/dev/null; then
|
||||||
|
remote_sha=$(git rev-parse "$remote_ref")
|
||||||
|
git push --force-with-lease="refs/heads/$UPDATE_BRANCH:$remote_sha" origin "HEAD:refs/heads/$UPDATE_BRANCH"
|
||||||
|
else
|
||||||
|
git push origin "HEAD:refs/heads/$UPDATE_BRANCH"
|
||||||
|
fi
|
||||||
|
|
||||||
|
body_file="$RUNNER_TEMP/georelay-pr-body.md"
|
||||||
|
{
|
||||||
|
echo "## Automated GeoRelay data proposal"
|
||||||
|
echo
|
||||||
|
echo "- Source: $SOURCE_URL"
|
||||||
|
echo "- Upstream commit: $SOURCE_COMMIT"
|
||||||
|
echo "- Data rows: $DATA_ROWS"
|
||||||
|
echo "- Unique normalized relays: $UNIQUE_RELAYS"
|
||||||
|
echo "- SHA-256: $DATA_SHA256"
|
||||||
|
echo
|
||||||
|
echo "The candidate passed strict UTF-8, schema, size, row-count, secure-host, coordinate, duplicate-conflict, and baseline-delta validation."
|
||||||
|
echo
|
||||||
|
echo "This PR is intentionally not auto-merged. Review the relay additions/removals before merging."
|
||||||
|
} > "$body_file"
|
||||||
|
|
||||||
|
existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty')
|
||||||
|
pr_error="$RUNNER_TEMP/georelay-pr-error.txt"
|
||||||
|
pr_url=""
|
||||||
|
if [[ -n "$existing_pr" ]]; then
|
||||||
|
if gh pr edit "$existing_pr" --repo "$GITHUB_REPOSITORY" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"; then
|
||||||
|
pr_url=$(gh pr view "$existing_pr" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if created_pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$UPDATE_BRANCH" --title "Update reviewed GeoRelay directory" --body-file "$body_file" 2> "$pr_error"); then
|
||||||
|
pr_url="$created_pr_url"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number")
|
||||||
|
tracking_issues=()
|
||||||
|
if [[ -n "$tracking_issue_numbers" ]]; then
|
||||||
|
mapfile -t tracking_issues <<< "$tracking_issue_numbers"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$pr_url" ]]; then
|
||||||
|
for issue_number in "${tracking_issues[@]}"; do
|
||||||
|
gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "A pull request is now available at $pr_url; closing this fallback tracking issue."
|
||||||
|
done
|
||||||
|
echo "Published GeoRelay review PR: $pr_url" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "::warning::GITHUB_TOKEN could not create or update the GeoRelay pull request; publishing the issues-write fallback."
|
||||||
|
if [[ -s "$pr_error" ]]; then
|
||||||
|
cat "$pr_error" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
compare_url="https://github.com/${GITHUB_REPOSITORY}/compare/main...${UPDATE_BRANCH}?expand=1"
|
||||||
|
issue_body_file="$RUNNER_TEMP/georelay-tracking-issue-body.md"
|
||||||
|
{
|
||||||
|
echo "## Validated GeoRelay update awaiting review"
|
||||||
|
echo
|
||||||
|
echo "The automation branch was updated, but this workflow token could not create or update the pull request. Use the compare link below to create it manually."
|
||||||
|
echo
|
||||||
|
echo "- Compare and create PR: $compare_url"
|
||||||
|
echo "- Automation branch: $UPDATE_BRANCH"
|
||||||
|
echo "- Source: $SOURCE_URL"
|
||||||
|
echo "- Upstream commit: $SOURCE_COMMIT"
|
||||||
|
echo "- Data rows: $DATA_ROWS"
|
||||||
|
echo "- Unique normalized relays: $UNIQUE_RELAYS"
|
||||||
|
echo "- SHA-256: $DATA_SHA256"
|
||||||
|
echo
|
||||||
|
echo "The snapshot passed the repository's strict validator before the branch was pushed."
|
||||||
|
} > "$issue_body_file"
|
||||||
|
|
||||||
|
if (( ${#tracking_issues[@]} > 0 )); then
|
||||||
|
primary_issue="${tracking_issues[0]}"
|
||||||
|
gh issue edit "$primary_issue" --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file"
|
||||||
|
issue_url=$(gh issue view "$primary_issue" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||||
|
for duplicate_issue in "${tracking_issues[@]:1}"; do
|
||||||
|
gh issue close "$duplicate_issue" --repo "$GITHUB_REPOSITORY" --comment "Closing duplicate GeoRelay automation tracking issue; #$primary_issue is canonical."
|
||||||
|
done
|
||||||
|
else
|
||||||
|
issue_url=$(gh issue create --repo "$GITHUB_REPOSITORY" --title "$TRACKING_ISSUE_TITLE" --body-file "$issue_body_file")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Do not claim success until the fallback issue was confirmed.
|
||||||
|
[[ -n "$issue_url" ]]
|
||||||
|
echo "Published GeoRelay tracking issue fallback: $issue_url" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
|
- name: Clean obsolete automation review state
|
||||||
|
if: steps.changes.outputs.changed == 'false'
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
gh auth setup-git
|
||||||
|
|
||||||
|
existing_pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --base main --head "$UPDATE_BRANCH" --json number --jq '.[0].number // empty')
|
||||||
|
if [[ -n "$existing_pr" ]]; then
|
||||||
|
gh pr close "$existing_pr" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation proposal."
|
||||||
|
echo "Closed obsolete PR #$existing_pr." >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
fi
|
||||||
|
|
||||||
|
tracking_issue_numbers=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$TRACKING_ISSUE_TITLE\" in:title" --limit 100 --json number,title --jq ".[] | select(.title == \"$TRACKING_ISSUE_TITLE\") | .number")
|
||||||
|
if [[ -n "$tracking_issue_numbers" ]]; then
|
||||||
|
while IFS= read -r issue_number; do
|
||||||
|
gh issue close "$issue_number" --repo "$GITHUB_REPOSITORY" --comment "Upstream now matches the reviewed file on main; closing this obsolete automation tracker."
|
||||||
|
echo "Closed obsolete tracking issue #$issue_number." >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
done <<< "$tracking_issue_numbers"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if git ls-remote --exit-code --heads origin "refs/heads/$UPDATE_BRANCH" > /dev/null; then
|
||||||
|
git push origin --delete "$UPDATE_BRANCH"
|
||||||
|
echo "Deleted obsolete automation branch $UPDATE_BRANCH." >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
else
|
||||||
|
ls_remote_status=$?
|
||||||
|
if (( ls_remote_status != 2 )); then
|
||||||
|
echo "::error::Could not inspect the obsolete automation branch"
|
||||||
|
exit "$ls_remote_status"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|||||||
@@ -32,6 +32,23 @@ struct GeoRelayDirectoryDependencies {
|
|||||||
var retrySleep: (TimeInterval) async -> Void
|
var retrySleep: (TimeInterval) async -> Void
|
||||||
var activeNotificationName: Notification.Name?
|
var activeNotificationName: Notification.Name?
|
||||||
var autoStart: Bool
|
var autoStart: Bool
|
||||||
|
var validationPolicy: GeoRelayDirectoryValidationPolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GeoRelayDirectoryValidationPolicy: Sendable {
|
||||||
|
let maximumBytes: Int
|
||||||
|
let maximumRows: Int
|
||||||
|
let maximumEntries: Int
|
||||||
|
let minimumRemoteEntries: Int
|
||||||
|
let minimumRetainedFraction: Double
|
||||||
|
|
||||||
|
static let live = GeoRelayDirectoryValidationPolicy(
|
||||||
|
maximumBytes: 512 * 1024,
|
||||||
|
maximumRows: 5_000,
|
||||||
|
maximumEntries: 5_000,
|
||||||
|
minimumRemoteEntries: 50,
|
||||||
|
minimumRetainedFraction: 0.5
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
private extension GeoRelayDirectoryDependencies {
|
private extension GeoRelayDirectoryDependencies {
|
||||||
@@ -44,12 +61,16 @@ private extension GeoRelayDirectoryDependencies {
|
|||||||
#else
|
#else
|
||||||
let activeNotificationName: Notification.Name? = nil
|
let activeNotificationName: Notification.Name? = nil
|
||||||
#endif
|
#endif
|
||||||
|
let validationPolicy = GeoRelayDirectoryValidationPolicy.live
|
||||||
|
|
||||||
return Self(
|
return Self(
|
||||||
userDefaults: .standard,
|
userDefaults: .standard,
|
||||||
notificationCenter: .default,
|
notificationCenter: .default,
|
||||||
now: Date.init,
|
now: Date.init,
|
||||||
remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/georelays/refs/heads/main/nostr_relays.csv")!,
|
// Runtime refreshes only from bitchat's reviewed copy. Upstream
|
||||||
|
// georelays/main is imported by a validator-backed pull request,
|
||||||
|
// so an upstream mutation cannot immediately retarget clients.
|
||||||
|
remoteURL: URL(string: "https://raw.githubusercontent.com/permissionlesstech/bitchat/refs/heads/main/relays/online_relays_gps.csv")!,
|
||||||
fetchInterval: TransportConfig.geoRelayFetchIntervalSeconds,
|
fetchInterval: TransportConfig.geoRelayFetchIntervalSeconds,
|
||||||
refreshCheckInterval: TransportConfig.geoRelayRefreshCheckIntervalSeconds,
|
refreshCheckInterval: TransportConfig.geoRelayRefreshCheckIntervalSeconds,
|
||||||
retryInitialSeconds: TransportConfig.geoRelayRetryInitialSeconds,
|
retryInitialSeconds: TransportConfig.geoRelayRetryInitialSeconds,
|
||||||
@@ -58,7 +79,27 @@ private extension GeoRelayDirectoryDependencies {
|
|||||||
makeFetchData: {
|
makeFetchData: {
|
||||||
let session = TorURLSession.shared.session
|
let session = TorURLSession.shared.session
|
||||||
return { request in
|
return { request in
|
||||||
let (data, _) = try await session.data(for: request)
|
let (bytes, response) = try await session.bytes(for: request)
|
||||||
|
guard let response = response as? HTTPURLResponse,
|
||||||
|
(200...299).contains(response.statusCode),
|
||||||
|
response.url == request.url else {
|
||||||
|
throw URLError(.badServerResponse)
|
||||||
|
}
|
||||||
|
|
||||||
|
let maximumBytes = validationPolicy.maximumBytes
|
||||||
|
guard response.expectedContentLength <= Int64(maximumBytes) else {
|
||||||
|
throw URLError(.dataLengthExceedsMaximum)
|
||||||
|
}
|
||||||
|
var data = Data()
|
||||||
|
if response.expectedContentLength > 0 {
|
||||||
|
data.reserveCapacity(Int(response.expectedContentLength))
|
||||||
|
}
|
||||||
|
for try await byte in bytes {
|
||||||
|
guard data.count < maximumBytes else {
|
||||||
|
throw URLError(.dataLengthExceedsMaximum)
|
||||||
|
}
|
||||||
|
data.append(byte)
|
||||||
|
}
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -76,7 +117,11 @@ private extension GeoRelayDirectoryDependencies {
|
|||||||
)
|
)
|
||||||
let dir = base.appendingPathComponent("bitchat", isDirectory: true)
|
let dir = base.appendingPathComponent("bitchat", isDirectory: true)
|
||||||
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
||||||
return dir.appendingPathComponent("georelays_cache.csv")
|
// v2 ignores caches populated from the old direct-upstream
|
||||||
|
// trust path and subjects every load to strict validation.
|
||||||
|
let legacyCache = dir.appendingPathComponent("georelays_cache.csv")
|
||||||
|
try? FileManager.default.removeItem(at: legacyCache)
|
||||||
|
return dir.appendingPathComponent("georelays_cache_v2.csv")
|
||||||
} catch {
|
} catch {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -94,7 +139,8 @@ private extension GeoRelayDirectoryDependencies {
|
|||||||
try? await Task.sleep(nanoseconds: nanoseconds)
|
try? await Task.sleep(nanoseconds: nanoseconds)
|
||||||
},
|
},
|
||||||
activeNotificationName: activeNotificationName,
|
activeNotificationName: activeNotificationName,
|
||||||
autoStart: true
|
autoStart: true,
|
||||||
|
validationPolicy: validationPolicy
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -125,7 +171,7 @@ final class GeoRelayDirectory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private enum DetachedFetchOutcome: Sendable {
|
private enum DetachedFetchOutcome: Sendable {
|
||||||
case success(entries: [Entry], csv: String)
|
case success(entries: [Entry], csv: Data)
|
||||||
case torNotReady
|
case torNotReady
|
||||||
case invalidData
|
case invalidData
|
||||||
case network(String)
|
case network(String)
|
||||||
@@ -212,6 +258,8 @@ final class GeoRelayDirectory {
|
|||||||
)
|
)
|
||||||
let awaitTorReady = dependencies.awaitTorReady
|
let awaitTorReady = dependencies.awaitTorReady
|
||||||
let fetchData = dependencies.makeFetchData()
|
let fetchData = dependencies.makeFetchData()
|
||||||
|
let validationPolicy = dependencies.validationPolicy
|
||||||
|
let baselineEntries = Set(entries)
|
||||||
|
|
||||||
Task { [weak self] in
|
Task { [weak self] in
|
||||||
guard let self else { return }
|
guard let self else { return }
|
||||||
@@ -219,7 +267,9 @@ final class GeoRelayDirectory {
|
|||||||
let outcome = await Self.fetchRemoteOutcome(
|
let outcome = await Self.fetchRemoteOutcome(
|
||||||
request: request,
|
request: request,
|
||||||
awaitTorReady: awaitTorReady,
|
awaitTorReady: awaitTorReady,
|
||||||
fetchData: fetchData
|
fetchData: fetchData,
|
||||||
|
validationPolicy: validationPolicy,
|
||||||
|
baselineEntries: baselineEntries
|
||||||
)
|
)
|
||||||
|
|
||||||
switch outcome {
|
switch outcome {
|
||||||
@@ -238,7 +288,9 @@ final class GeoRelayDirectory {
|
|||||||
nonisolated private static func fetchRemoteOutcome(
|
nonisolated private static func fetchRemoteOutcome(
|
||||||
request: URLRequest,
|
request: URLRequest,
|
||||||
awaitTorReady: @escaping @Sendable () async -> Bool,
|
awaitTorReady: @escaping @Sendable () async -> Bool,
|
||||||
fetchData: @escaping @Sendable (URLRequest) async throws -> Data
|
fetchData: @escaping @Sendable (URLRequest) async throws -> Data,
|
||||||
|
validationPolicy: GeoRelayDirectoryValidationPolicy,
|
||||||
|
baselineEntries: Set<Entry>
|
||||||
) async -> DetachedFetchOutcome {
|
) async -> DetachedFetchOutcome {
|
||||||
await Task.detached(priority: .utility) {
|
await Task.detached(priority: .utility) {
|
||||||
let ready = await awaitTorReady()
|
let ready = await awaitTorReady()
|
||||||
@@ -246,16 +298,16 @@ final class GeoRelayDirectory {
|
|||||||
|
|
||||||
do {
|
do {
|
||||||
let data = try await fetchData(request)
|
let data = try await fetchData(request)
|
||||||
guard let text = String(data: data, encoding: .utf8) else {
|
guard let parsed = Self.validatedEntries(
|
||||||
|
from: data,
|
||||||
|
policy: validationPolicy,
|
||||||
|
minimumEntries: validationPolicy.minimumRemoteEntries,
|
||||||
|
baselineEntries: baselineEntries
|
||||||
|
) else {
|
||||||
return .invalidData
|
return .invalidData
|
||||||
}
|
}
|
||||||
|
|
||||||
let parsed = Self.parseCSV(text)
|
return .success(entries: parsed, csv: data)
|
||||||
guard !parsed.isEmpty else {
|
|
||||||
return .invalidData
|
|
||||||
}
|
|
||||||
|
|
||||||
return .success(entries: parsed, csv: text)
|
|
||||||
} catch {
|
} catch {
|
||||||
return .network(error.localizedDescription)
|
return .network(error.localizedDescription)
|
||||||
}
|
}
|
||||||
@@ -269,7 +321,7 @@ final class GeoRelayDirectory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@MainActor
|
@MainActor
|
||||||
private func handleFetchSuccess(entries parsed: [Entry], csv: String) {
|
private func handleFetchSuccess(entries parsed: [Entry], csv: Data) {
|
||||||
entries = parsed
|
entries = parsed
|
||||||
persistCache(csv)
|
persistCache(csv)
|
||||||
dependencies.userDefaults.set(dependencies.now(), forKey: lastFetchKey)
|
dependencies.userDefaults.set(dependencies.now(), forKey: lastFetchKey)
|
||||||
@@ -321,9 +373,8 @@ final class GeoRelayDirectory {
|
|||||||
cleanupState.retryTask = nil
|
cleanupState.retryTask = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
private func persistCache(_ text: String) {
|
private func persistCache(_ data: Data) {
|
||||||
guard let url = dependencies.cacheURL() else { return }
|
guard let url = dependencies.cacheURL() else { return }
|
||||||
guard let data = text.data(using: .utf8) else { return }
|
|
||||||
do {
|
do {
|
||||||
try dependencies.writeData(data, url)
|
try dependencies.writeData(data, url)
|
||||||
} catch {
|
} catch {
|
||||||
@@ -336,9 +387,12 @@ final class GeoRelayDirectory {
|
|||||||
// Prefer cached file if present
|
// Prefer cached file if present
|
||||||
if let cache = dependencies.cacheURL(),
|
if let cache = dependencies.cacheURL(),
|
||||||
let data = dependencies.readData(cache),
|
let data = dependencies.readData(cache),
|
||||||
let text = String(data: data, encoding: .utf8) {
|
let entries = Self.validatedEntries(
|
||||||
let arr = Self.parseCSV(text)
|
from: data,
|
||||||
if !arr.isEmpty { return arr }
|
policy: dependencies.validationPolicy,
|
||||||
|
minimumEntries: 1
|
||||||
|
) {
|
||||||
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try bundled resource(s)
|
// Try bundled resource(s)
|
||||||
@@ -346,36 +400,157 @@ final class GeoRelayDirectory {
|
|||||||
|
|
||||||
for url in bundleCandidates {
|
for url in bundleCandidates {
|
||||||
if let data = dependencies.readData(url),
|
if let data = dependencies.readData(url),
|
||||||
let text = String(data: data, encoding: .utf8) {
|
let entries = Self.validatedEntries(
|
||||||
let arr = Self.parseCSV(text)
|
from: data,
|
||||||
if !arr.isEmpty { return arr }
|
policy: dependencies.validationPolicy,
|
||||||
|
minimumEntries: 1
|
||||||
|
) {
|
||||||
|
return entries
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try filesystem path (development/test)
|
// Try filesystem path (development/test)
|
||||||
if let cwd = dependencies.currentDirectoryPath(),
|
if let cwd = dependencies.currentDirectoryPath(),
|
||||||
let data = dependencies.readData(URL(fileURLWithPath: cwd).appendingPathComponent("relays/online_relays_gps.csv")),
|
let data = dependencies.readData(URL(fileURLWithPath: cwd).appendingPathComponent("relays/online_relays_gps.csv")),
|
||||||
let text = String(data: data, encoding: .utf8) {
|
let entries = Self.validatedEntries(
|
||||||
return Self.parseCSV(text)
|
from: data,
|
||||||
|
policy: dependencies.validationPolicy,
|
||||||
|
minimumEntries: 1
|
||||||
|
) {
|
||||||
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
SecureLogger.warning("GeoRelayDirectory: no local CSV found; entries empty", category: .session)
|
SecureLogger.warning("GeoRelayDirectory: no local CSV found; entries empty", category: .session)
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
|
|
||||||
nonisolated static func parseCSV(_ text: String) -> [Entry] {
|
/// Parses the fixed three-column format as an all-or-nothing trust unit.
|
||||||
var result: Set<Entry> = []
|
/// One malformed or conflicting row rejects the complete dataset rather
|
||||||
let lines = text.split(whereSeparator: { $0.isNewline })
|
/// than silently shrinking or partially replacing the current directory.
|
||||||
for (idx, raw) in lines.enumerated() {
|
nonisolated static func validatedEntries(
|
||||||
guard let line = raw.trimmedOrNilIfEmpty else { continue }
|
from data: Data,
|
||||||
if idx == 0 && line.lowercased().contains("relay url") { continue }
|
policy: GeoRelayDirectoryValidationPolicy,
|
||||||
let parts = line.split(separator: ",").map { $0.trimmed }
|
minimumEntries: Int,
|
||||||
guard parts.count >= 3 else { continue }
|
baselineEntries: Set<Entry>? = nil
|
||||||
guard let host = NostrRelayURL.directoryAddress(parts[0]) else { continue }
|
) -> [Entry]? {
|
||||||
guard let lat = Double(parts[1]), let lon = Double(parts[2]) else { continue }
|
guard !data.isEmpty, data.count <= policy.maximumBytes,
|
||||||
result.insert(Entry(host: host, lat: lat, lon: lon))
|
let text = String(data: data, encoding: .utf8),
|
||||||
|
!text.hasPrefix("\u{feff}") else {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
return Array(result)
|
|
||||||
|
let lines = text.split(whereSeparator: { $0.isNewline })
|
||||||
|
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||||
|
.filter { !$0.isEmpty }
|
||||||
|
guard let header = lines.first,
|
||||||
|
lines.count - 1 <= policy.maximumRows else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let headerParts = header
|
||||||
|
.split(separator: ",", omittingEmptySubsequences: false)
|
||||||
|
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }
|
||||||
|
let supportedHeaders = [
|
||||||
|
["relay url", "latitude", "longitude"],
|
||||||
|
["relay url", "lat", "lon"]
|
||||||
|
]
|
||||||
|
guard supportedHeaders.contains(headerParts) else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var entriesByHost: [String: Entry] = [:]
|
||||||
|
for line in lines.dropFirst() {
|
||||||
|
let parts = line
|
||||||
|
.split(separator: ",", omittingEmptySubsequences: false)
|
||||||
|
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||||
|
guard parts.count == 3,
|
||||||
|
let host = validatedDirectoryAddress(parts[0]),
|
||||||
|
let latitude = Double(parts[1]), latitude.isFinite,
|
||||||
|
(-90.0...90.0).contains(latitude),
|
||||||
|
let longitude = Double(parts[2]), longitude.isFinite,
|
||||||
|
(-180.0...180.0).contains(longitude) else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let entry = Entry(host: host, lat: latitude, lon: longitude)
|
||||||
|
if let existing = entriesByHost[host], existing != entry {
|
||||||
|
// One endpoint cannot truthfully occupy two coordinates. Do
|
||||||
|
// not let row ordering choose which location clients trust.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
entriesByHost[host] = entry
|
||||||
|
guard entriesByHost.count <= policy.maximumEntries else { return nil }
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsedEntries = Set(entriesByHost.values)
|
||||||
|
guard parsedEntries.count >= minimumEntries else { return nil }
|
||||||
|
|
||||||
|
if let baselineEntries {
|
||||||
|
guard (0...1).contains(policy.minimumRetainedFraction) else { return nil }
|
||||||
|
let requiredOverlap = Int(
|
||||||
|
ceil(Double(baselineEntries.count) * policy.minimumRetainedFraction)
|
||||||
|
)
|
||||||
|
guard parsedEntries.intersection(baselineEntries).count >= requiredOverlap else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedEntries.sorted {
|
||||||
|
($0.host, $0.lat, $0.lon) < ($1.host, $1.lat, $1.lon)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nonisolated private static func validatedDirectoryAddress(_ rawValue: String) -> String? {
|
||||||
|
let value = rawValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||||
|
guard !value.isEmpty,
|
||||||
|
value.unicodeScalars.allSatisfy({
|
||||||
|
$0.isASCII && !CharacterSet.controlCharacters.contains($0)
|
||||||
|
}) else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let candidate = value.contains("://") ? value : "wss://\(value)"
|
||||||
|
guard let components = URLComponents(string: candidate),
|
||||||
|
let scheme = components.scheme?.lowercased(),
|
||||||
|
scheme == "wss" || scheme == "https",
|
||||||
|
components.user == nil,
|
||||||
|
components.password == nil,
|
||||||
|
components.query == nil,
|
||||||
|
components.fragment == nil,
|
||||||
|
components.path.isEmpty || components.path == "/",
|
||||||
|
let rawHost = components.host else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let host = rawHost.lowercased()
|
||||||
|
guard !host.isEmpty, host.count <= 253,
|
||||||
|
host.unicodeScalars.allSatisfy({ $0.isASCII }),
|
||||||
|
!host.hasSuffix("."),
|
||||||
|
host != "localhost",
|
||||||
|
!host.hasSuffix(".localhost"),
|
||||||
|
!host.hasSuffix(".local"),
|
||||||
|
!host.hasSuffix(".internal") else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let labels = host.split(separator: ".", omittingEmptySubsequences: false)
|
||||||
|
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyz0123456789-")
|
||||||
|
guard labels.count >= 2,
|
||||||
|
!labels.allSatisfy({ $0.allSatisfy(\.isNumber) }),
|
||||||
|
labels.allSatisfy({ label in
|
||||||
|
(1...63).contains(label.count) &&
|
||||||
|
label.first != "-" &&
|
||||||
|
label.last != "-" &&
|
||||||
|
label.unicodeScalars.allSatisfy { allowed.contains($0) }
|
||||||
|
}) else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if let port = components.port {
|
||||||
|
guard (1...65_535).contains(port) else { return nil }
|
||||||
|
if port != 443 { return "\(host):\(port)" }
|
||||||
|
}
|
||||||
|
return host
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Observers & Timers
|
// MARK: - Observers & Timers
|
||||||
|
|||||||
@@ -39,13 +39,4 @@ enum NostrRelayURL {
|
|||||||
|
|
||||||
return components.string
|
return components.string
|
||||||
}
|
}
|
||||||
|
|
||||||
static func directoryAddress(_ rawValue: String) -> String? {
|
|
||||||
guard var normalized = normalized(rawValue, defaultScheme: "wss") else { return nil }
|
|
||||||
for prefix in ["wss://", "ws://"] where normalized.hasPrefix(prefix) {
|
|
||||||
normalized.removeFirst(prefix.count)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return normalized
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,19 +5,25 @@ import XCTest
|
|||||||
|
|
||||||
@MainActor
|
@MainActor
|
||||||
final class GeoRelayDirectoryTests: XCTestCase {
|
final class GeoRelayDirectoryTests: XCTestCase {
|
||||||
func test_parseCSV_normalizesRelaySchemesAndDeduplicatesEntries() {
|
private func parse(_ csv: String) -> [GeoRelayDirectory.Entry] {
|
||||||
|
GeoRelayDirectory.validatedEntries(
|
||||||
|
from: Data(csv.utf8),
|
||||||
|
policy: .live,
|
||||||
|
minimumEntries: 1
|
||||||
|
) ?? []
|
||||||
|
}
|
||||||
|
|
||||||
|
func test_parseCSV_normalizesSecureRelaySchemesAndDeduplicatesEntries() {
|
||||||
let csv = """
|
let csv = """
|
||||||
relay url,lat,lon
|
relay url,lat,lon
|
||||||
wss://one.example/,10,20
|
wss://one.example/,10,20
|
||||||
https://one.example,10,20
|
https://one.example,10,20
|
||||||
wss://one.example:443/,10,20
|
wss://one.example:443/,10,20
|
||||||
http://two.example/,11,21
|
two.example,11,21
|
||||||
wss://two.example:443,11,21
|
wss://two.example:443,11,21
|
||||||
invalid row
|
|
||||||
ws://three.example,not-a-lat,22
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
let parsed = Set(GeoRelayDirectory.parseCSV(csv))
|
let parsed = Set(parse(csv))
|
||||||
|
|
||||||
XCTAssertEqual(
|
XCTAssertEqual(
|
||||||
parsed,
|
parsed,
|
||||||
@@ -28,6 +34,136 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func test_parseCSV_rejectsWholeDatasetWhenAnyRowOrHeaderIsUnsafe() {
|
||||||
|
let invalidCSVs = [
|
||||||
|
"relay,lat,lon\nrelay.example,1,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,1\n",
|
||||||
|
"relay url,lat,lon\nhttp://relay.example,1,2\n",
|
||||||
|
"relay url,lat,lon\nwss://user@relay.example,1,2\n",
|
||||||
|
"relay url,lat,lon\nwss://relay.example/path,1,2\n",
|
||||||
|
"relay url,lat,lon\nwss://relay.example?,1,2\n",
|
||||||
|
"relay url,lat,lon\nwss://relay.example#,1,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example:0,1,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example:99999,1,2\n",
|
||||||
|
"relay url,lat,lon\nlocalhost,1,2\n",
|
||||||
|
"relay url,lat,lon\nr\u{00e9}lay.example,1,2\n",
|
||||||
|
"relay url,lat,lon\nrelay\u{202e}.example,1,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,NaN,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,1_0,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,\u{0661}\u{0660},2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,\u{ff11}\u{ff10},2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,91,2\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,1,181\n",
|
||||||
|
"relay url,lat,lon\nrelay.example,1,2\nrelay.example,3,4\n"
|
||||||
|
]
|
||||||
|
|
||||||
|
for csv in invalidCSVs {
|
||||||
|
XCTAssertTrue(parse(csv).isEmpty, csv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func test_validatedEntries_enforcesByteRowEntryAndRetentionLimits() {
|
||||||
|
let restrictive = GeoRelayDirectoryValidationPolicy(
|
||||||
|
maximumBytes: 100,
|
||||||
|
maximumRows: 2,
|
||||||
|
maximumEntries: 2,
|
||||||
|
minimumRemoteEntries: 1,
|
||||||
|
minimumRetainedFraction: 0.5
|
||||||
|
)
|
||||||
|
let one = Data("relay url,lat,lon\none.example,1,2\n".utf8)
|
||||||
|
let three = Data("relay url,lat,lon\none.example,1,2\ntwo.example,3,4\nthree.example,5,6\n".utf8)
|
||||||
|
|
||||||
|
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: one,
|
||||||
|
policy: restrictive,
|
||||||
|
minimumEntries: 2
|
||||||
|
))
|
||||||
|
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: Data(repeating: 0x41, count: 101),
|
||||||
|
policy: restrictive,
|
||||||
|
minimumEntries: 1
|
||||||
|
))
|
||||||
|
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: three,
|
||||||
|
policy: restrictive,
|
||||||
|
minimumEntries: 1
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
func test_validatedEntries_requiresExactBaselineEntryOverlap() throws {
|
||||||
|
let policy = GeoRelayDirectoryValidationPolicy(
|
||||||
|
maximumBytes: 1_000,
|
||||||
|
maximumRows: 10,
|
||||||
|
maximumEntries: 10,
|
||||||
|
minimumRemoteEntries: 1,
|
||||||
|
minimumRetainedFraction: 0.5
|
||||||
|
)
|
||||||
|
let baseline = Set(try XCTUnwrap(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: Data("""
|
||||||
|
relay url,lat,lon
|
||||||
|
one.example,1,1
|
||||||
|
two.example,2,2
|
||||||
|
three.example,3,3
|
||||||
|
""".utf8),
|
||||||
|
policy: policy,
|
||||||
|
minimumEntries: 1
|
||||||
|
)))
|
||||||
|
let disjoint = Data("""
|
||||||
|
relay url,lat,lon
|
||||||
|
four.example,1,1
|
||||||
|
five.example,2,2
|
||||||
|
six.example,3,3
|
||||||
|
""".utf8)
|
||||||
|
let rewrittenCoordinates = Data("""
|
||||||
|
relay url,lat,lon
|
||||||
|
one.example,11,11
|
||||||
|
two.example,12,12
|
||||||
|
three.example,13,13
|
||||||
|
""".utf8)
|
||||||
|
let halfRetained = Data("""
|
||||||
|
relay url,lat,lon
|
||||||
|
wss://one.example:443/,1,1
|
||||||
|
https://two.example/,2,2
|
||||||
|
replacement.example,4,4
|
||||||
|
""".utf8)
|
||||||
|
|
||||||
|
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: disjoint,
|
||||||
|
policy: policy,
|
||||||
|
minimumEntries: 1,
|
||||||
|
baselineEntries: baseline
|
||||||
|
))
|
||||||
|
XCTAssertNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: rewrittenCoordinates,
|
||||||
|
policy: policy,
|
||||||
|
minimumEntries: 1,
|
||||||
|
baselineEntries: baseline
|
||||||
|
))
|
||||||
|
XCTAssertNotNil(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: halfRetained,
|
||||||
|
policy: policy,
|
||||||
|
minimumEntries: 1,
|
||||||
|
baselineEntries: baseline
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
func test_bundledReviewedCSV_passesStrictProductionValidation() throws {
|
||||||
|
let repositoryRoot = URL(fileURLWithPath: #filePath)
|
||||||
|
.deletingLastPathComponent()
|
||||||
|
.deletingLastPathComponent()
|
||||||
|
.deletingLastPathComponent()
|
||||||
|
let data = try Data(
|
||||||
|
contentsOf: repositoryRoot.appendingPathComponent("relays/online_relays_gps.csv")
|
||||||
|
)
|
||||||
|
|
||||||
|
let entries = try XCTUnwrap(GeoRelayDirectory.validatedEntries(
|
||||||
|
from: data,
|
||||||
|
policy: .live,
|
||||||
|
minimumEntries: GeoRelayDirectoryValidationPolicy.live.minimumRemoteEntries
|
||||||
|
))
|
||||||
|
XCTAssertGreaterThan(entries.count, 250)
|
||||||
|
}
|
||||||
|
|
||||||
func test_closestRelays_sortsByDistanceForLatLonAndGeohash() {
|
func test_closestRelays_sortsByDistanceForLatLonAndGeohash() {
|
||||||
let harness = makeHarness(
|
let harness = makeHarness(
|
||||||
cacheCSV: """
|
cacheCSV: """
|
||||||
@@ -243,6 +379,53 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
|||||||
XCTAssertFalse(directory.debugHasRetryTask)
|
XCTAssertFalse(directory.debugHasRetryTask)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func test_prefetchIfNeeded_rejectsSharpValidLookingTruncationBeforeCaching() async {
|
||||||
|
let cached = """
|
||||||
|
relay url,lat,lon
|
||||||
|
old-one.example,1,1
|
||||||
|
old-two.example,2,2
|
||||||
|
old-three.example,3,3
|
||||||
|
"""
|
||||||
|
let truncated = """
|
||||||
|
relay url,lat,lon
|
||||||
|
attacker.example,9,9
|
||||||
|
"""
|
||||||
|
let recovered = """
|
||||||
|
relay url,lat,lon
|
||||||
|
old-one.example,1,1
|
||||||
|
old-two.example,2,2
|
||||||
|
new-three.example,6,6
|
||||||
|
"""
|
||||||
|
let harness = makeHarness(
|
||||||
|
cacheCSV: cached,
|
||||||
|
fetchResults: [
|
||||||
|
.success(Data(truncated.utf8)),
|
||||||
|
.success(Data(recovered.utf8))
|
||||||
|
],
|
||||||
|
validationPolicy: GeoRelayDirectoryValidationPolicy(
|
||||||
|
maximumBytes: 64 * 1024,
|
||||||
|
maximumRows: 1_000,
|
||||||
|
maximumEntries: 1_000,
|
||||||
|
minimumRemoteEntries: 1,
|
||||||
|
minimumRetainedFraction: 0.5
|
||||||
|
)
|
||||||
|
)
|
||||||
|
let directory = GeoRelayDirectory(dependencies: harness.dependencies)
|
||||||
|
|
||||||
|
directory.prefetchIfNeeded()
|
||||||
|
|
||||||
|
let refreshed = await waitUntil {
|
||||||
|
directory.entries.contains(where: { $0.host == "new-three.example" })
|
||||||
|
}
|
||||||
|
XCTAssertTrue(refreshed)
|
||||||
|
XCTAssertFalse(directory.entries.contains(where: { $0.host == "attacker.example" }))
|
||||||
|
let requestCount = await harness.fetcher.recordedRequestCount()
|
||||||
|
let retryDelays = await harness.retryRecorder.recordedDelays()
|
||||||
|
XCTAssertEqual(requestCount, 2)
|
||||||
|
XCTAssertEqual(retryDelays, [5])
|
||||||
|
XCTAssertEqual(harness.fileStore.dataByURL[harness.cacheURL], Data(recovered.utf8))
|
||||||
|
}
|
||||||
|
|
||||||
func test_observers_triggerPrefetchesForTorReadyAndAppActivation() async {
|
func test_observers_triggerPrefetchesForTorReadyAndAppActivation() async {
|
||||||
let activeNotification = Notification.Name("GeoRelayDirectoryTests.didBecomeActive")
|
let activeNotification = Notification.Name("GeoRelayDirectoryTests.didBecomeActive")
|
||||||
let harness = makeHarness(
|
let harness = makeHarness(
|
||||||
@@ -289,7 +472,14 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
|||||||
fetchFactoryObserver: (@MainActor @Sendable () -> Void)? = nil,
|
fetchFactoryObserver: (@MainActor @Sendable () -> Void)? = nil,
|
||||||
fetchObserver: (@Sendable () async -> Void)? = nil,
|
fetchObserver: (@Sendable () async -> Void)? = nil,
|
||||||
autoStart: Bool = false,
|
autoStart: Bool = false,
|
||||||
activeNotificationName: Notification.Name? = nil
|
activeNotificationName: Notification.Name? = nil,
|
||||||
|
validationPolicy: GeoRelayDirectoryValidationPolicy = GeoRelayDirectoryValidationPolicy(
|
||||||
|
maximumBytes: 64 * 1024,
|
||||||
|
maximumRows: 1_000,
|
||||||
|
maximumEntries: 1_000,
|
||||||
|
minimumRemoteEntries: 1,
|
||||||
|
minimumRetainedFraction: 0
|
||||||
|
)
|
||||||
) -> GeoRelayHarness {
|
) -> GeoRelayHarness {
|
||||||
let userDefaultsSuite = "GeoRelayDirectoryTests.\(UUID().uuidString)"
|
let userDefaultsSuite = "GeoRelayDirectoryTests.\(UUID().uuidString)"
|
||||||
let userDefaults = UserDefaults(suiteName: userDefaultsSuite)!
|
let userDefaults = UserDefaults(suiteName: userDefaultsSuite)!
|
||||||
@@ -347,7 +537,8 @@ final class GeoRelayDirectoryTests: XCTestCase {
|
|||||||
await retryRecorder.record(delay)
|
await retryRecorder.record(delay)
|
||||||
},
|
},
|
||||||
activeNotificationName: activeNotificationName,
|
activeNotificationName: activeNotificationName,
|
||||||
autoStart: autoStart
|
autoStart: autoStart,
|
||||||
|
validationPolicy: validationPolicy
|
||||||
)
|
)
|
||||||
|
|
||||||
return GeoRelayHarness(
|
return GeoRelayHarness(
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
WORKFLOW_PATH = REPOSITORY_ROOT / ".github/workflows/fetch_georelays.yml"
|
||||||
|
|
||||||
|
|
||||||
|
class FetchGeoRelaysWorkflowTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
def test_write_capable_checkout_action_is_immutable(self) -> None:
|
||||||
|
checkout = re.search(r"uses: actions/checkout@([0-9a-f]+)", self.workflow)
|
||||||
|
self.assertIsNotNone(checkout)
|
||||||
|
self.assertRegex(checkout.group(1), r"^[0-9a-f]{40}$")
|
||||||
|
self.assertIn("persist-credentials: false", self.workflow)
|
||||||
|
|
||||||
|
def test_pr_failure_has_single_issue_fallback_with_review_metadata(self) -> None:
|
||||||
|
required_fragments = [
|
||||||
|
"issues: write",
|
||||||
|
"TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request",
|
||||||
|
"gh pr create",
|
||||||
|
"gh issue create",
|
||||||
|
"gh issue edit",
|
||||||
|
"compare/main...${UPDATE_BRANCH}?expand=1",
|
||||||
|
"Upstream commit: $SOURCE_COMMIT",
|
||||||
|
"Data rows: $DATA_ROWS",
|
||||||
|
"Unique normalized relays: $UNIQUE_RELAYS",
|
||||||
|
"SHA-256: $DATA_SHA256",
|
||||||
|
'[[ -n "$issue_url" ]]',
|
||||||
|
]
|
||||||
|
for fragment in required_fragments:
|
||||||
|
with self.subTest(fragment=fragment):
|
||||||
|
self.assertIn(fragment, self.workflow)
|
||||||
|
|
||||||
|
confirmed = self.workflow.index('[[ -n "$issue_url" ]]')
|
||||||
|
success_summary = self.workflow.index(
|
||||||
|
"Published GeoRelay tracking issue fallback: $issue_url"
|
||||||
|
)
|
||||||
|
self.assertLess(confirmed, success_summary)
|
||||||
|
|
||||||
|
def test_obsolete_review_state_is_cleaned_without_pushing_main(self) -> None:
|
||||||
|
self.assertIn("gh pr close", self.workflow)
|
||||||
|
self.assertIn("gh issue close", self.workflow)
|
||||||
|
self.assertIn('git push origin --delete "$UPDATE_BRANCH"', self.workflow)
|
||||||
|
self.assertIn('git switch -C "$UPDATE_BRANCH"', self.workflow)
|
||||||
|
self.assertNotIn("git push origin main", self.workflow)
|
||||||
|
self.assertNotIn("git push --force origin main", self.workflow)
|
||||||
|
|
||||||
|
def test_workflow_runs_all_validator_tests(self) -> None:
|
||||||
|
self.assertIn(
|
||||||
|
'python3 -m unittest discover -s scripts/tests -p "test_*.py" -v',
|
||||||
|
self.workflow,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
import validate_georelays as validator
|
||||||
|
|
||||||
|
|
||||||
|
def csv_bytes(rows: list[str]) -> bytes:
|
||||||
|
return ("Relay URL,Latitude,Longitude\n" + "\n".join(rows) + "\n").encode()
|
||||||
|
|
||||||
|
|
||||||
|
class ValidateGeoRelaysTests(unittest.TestCase):
|
||||||
|
def test_validates_and_deduplicates_secure_relay_addresses(self) -> None:
|
||||||
|
data = csv_bytes(
|
||||||
|
[
|
||||||
|
"relay.example.com,10,20",
|
||||||
|
"wss://relay.example.com:443/,10,20",
|
||||||
|
"https://second.example.org,11,21",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = validator.validate_bytes(data, minimum_unique_relays=2)
|
||||||
|
|
||||||
|
self.assertEqual(summary.data_rows, 3)
|
||||||
|
self.assertEqual(summary.unique_relays, 2)
|
||||||
|
|
||||||
|
def test_rejects_insecure_or_non_host_relay_urls(self) -> None:
|
||||||
|
bad_addresses = [
|
||||||
|
"http://relay.example.com",
|
||||||
|
"ws://relay.example.com",
|
||||||
|
"wss://user@relay.example.com",
|
||||||
|
"wss://relay.example.com/path",
|
||||||
|
"wss://relay.example.com?",
|
||||||
|
"wss://relay.example.com#",
|
||||||
|
"relay.example.com:0",
|
||||||
|
"relay.example.com:99999",
|
||||||
|
"localhost",
|
||||||
|
"127.0.0.1",
|
||||||
|
"relay_example.com",
|
||||||
|
"relay\u202e.example.com",
|
||||||
|
]
|
||||||
|
|
||||||
|
for address in bad_addresses:
|
||||||
|
with self.subTest(address=address):
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_bytes(
|
||||||
|
csv_bytes([f"{address},10,20"]),
|
||||||
|
minimum_unique_relays=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_rejects_malformed_rows_and_unsafe_coordinates(self) -> None:
|
||||||
|
bad_rows = [
|
||||||
|
"relay.example.com,10",
|
||||||
|
"relay.example.com,NaN,20",
|
||||||
|
"relay.example.com,1_0,20",
|
||||||
|
"relay.example.com,\u0661\u0660,20",
|
||||||
|
"relay.example.com,\uff11\uff10,20",
|
||||||
|
"relay.example.com,91,20",
|
||||||
|
"relay.example.com,10,-181",
|
||||||
|
"relay.example.com,10,20,extra",
|
||||||
|
'"relay.example.com",10,20',
|
||||||
|
]
|
||||||
|
|
||||||
|
for row in bad_rows:
|
||||||
|
with self.subTest(row=row):
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_bytes(csv_bytes([row]), minimum_unique_relays=1)
|
||||||
|
|
||||||
|
def test_accepts_ascii_coordinate_forms_supported_by_swift_double(self) -> None:
|
||||||
|
summary = validator.validate_bytes(
|
||||||
|
csv_bytes(
|
||||||
|
[
|
||||||
|
"one.example.com,+1,-.5",
|
||||||
|
"two.example.com,1.e1,2E+1",
|
||||||
|
"three.example.com,01,20.",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
minimum_unique_relays=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(summary.unique_relays, 3)
|
||||||
|
|
||||||
|
def test_rejects_conflicts_limits_and_large_baseline_deltas(self) -> None:
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_bytes(
|
||||||
|
csv_bytes(["relay.example.com,10,20", "relay.example.com,11,21"]),
|
||||||
|
minimum_unique_relays=1,
|
||||||
|
)
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_bytes(b"x" * 20, maximum_bytes=10, minimum_unique_relays=1)
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_bytes(
|
||||||
|
csv_bytes(["one.example.com,1,1", "two.example.com,2,2"]),
|
||||||
|
minimum_unique_relays=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
baseline = csv_bytes(
|
||||||
|
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(120)]
|
||||||
|
)
|
||||||
|
shrunken = csv_bytes(
|
||||||
|
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(59)]
|
||||||
|
)
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_update(shrunken, baseline)
|
||||||
|
|
||||||
|
smaller_baseline = csv_bytes(
|
||||||
|
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||||
|
)
|
||||||
|
expanded = csv_bytes(
|
||||||
|
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(121)]
|
||||||
|
)
|
||||||
|
with self.assertRaises(validator.ValidationError):
|
||||||
|
validator.validate_update(expanded, smaller_baseline)
|
||||||
|
|
||||||
|
def test_update_requires_exact_normalized_baseline_entry_overlap(self) -> None:
|
||||||
|
baseline_rows = [
|
||||||
|
f"relay-{index}.example.com,{index % 80},{index % 170}"
|
||||||
|
for index in range(60)
|
||||||
|
]
|
||||||
|
baseline = csv_bytes(baseline_rows)
|
||||||
|
disjoint = csv_bytes(
|
||||||
|
[
|
||||||
|
f"attacker-{index}.example.com,{index % 80},{index % 170}"
|
||||||
|
for index in range(60)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
rewritten_coordinates = csv_bytes(
|
||||||
|
[
|
||||||
|
f"relay-{index}.example.com,{(index % 80) + 0.5},{index % 170}"
|
||||||
|
for index in range(60)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
for candidate in (disjoint, rewritten_coordinates):
|
||||||
|
with self.subTest(candidate=candidate[:80]):
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
validator.ValidationError,
|
||||||
|
"exact relay-coordinate entries",
|
||||||
|
):
|
||||||
|
validator.validate_update(candidate, baseline)
|
||||||
|
|
||||||
|
half_retained = csv_bytes(
|
||||||
|
[
|
||||||
|
f"wss://relay-{index}.example.com:443/,{index % 80},{index % 170}"
|
||||||
|
for index in range(30)
|
||||||
|
]
|
||||||
|
+ [
|
||||||
|
f"replacement-{index}.example.com,{index % 80},{index % 170}"
|
||||||
|
for index in range(30)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
summary = validator.validate_update(half_retained, baseline)
|
||||||
|
self.assertEqual(summary.unique_relays, 60)
|
||||||
|
|
||||||
|
def test_cli_copies_only_validated_data_and_emits_review_metadata(self) -> None:
|
||||||
|
rows = [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||||
|
data = csv_bytes(rows)
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
candidate = root / "candidate.csv"
|
||||||
|
baseline = root / "baseline.csv"
|
||||||
|
output = root / "output.csv"
|
||||||
|
github_output = root / "github-output.txt"
|
||||||
|
candidate.write_bytes(data)
|
||||||
|
baseline.write_bytes(data)
|
||||||
|
|
||||||
|
result = validator.main(
|
||||||
|
[
|
||||||
|
"--input", str(candidate),
|
||||||
|
"--baseline", str(baseline),
|
||||||
|
"--output", str(output),
|
||||||
|
"--github-output", str(github_output),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(result, 0)
|
||||||
|
self.assertEqual(output.read_bytes(), data)
|
||||||
|
metadata = github_output.read_text()
|
||||||
|
self.assertIn("unique_relays=60", metadata)
|
||||||
|
self.assertIn("sha256=", metadata)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,271 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Strict validator for the reviewed georelay CSV update workflow."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
MAX_BYTES = 512 * 1024
|
||||||
|
MAX_ROWS = 5_000
|
||||||
|
MAX_UNIQUE_RELAYS = 5_000
|
||||||
|
MIN_UNIQUE_RELAYS = 50
|
||||||
|
MIN_BASELINE_FRACTION = 0.5
|
||||||
|
MAX_BASELINE_MULTIPLIER = 2.0
|
||||||
|
EXPECTED_HEADER = ("relay url", "latitude", "longitude")
|
||||||
|
ASCII_DECIMAL_PATTERN = re.compile(
|
||||||
|
r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ValidationSummary:
|
||||||
|
data_rows: int
|
||||||
|
unique_relays: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _ValidatedDataset:
|
||||||
|
summary: ValidationSummary
|
||||||
|
entries: frozenset[tuple[str, float, float]]
|
||||||
|
|
||||||
|
|
||||||
|
def _has_disallowed_control(value: str) -> bool:
|
||||||
|
return any(
|
||||||
|
unicodedata.category(character) in {"Cc", "Cf"}
|
||||||
|
and character not in {"\r", "\n", "\t"}
|
||||||
|
for character in value
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_relay_address(raw_value: str) -> str:
|
||||||
|
value = raw_value.strip()
|
||||||
|
if not value or _has_disallowed_control(value):
|
||||||
|
raise ValidationError("relay address is empty or contains control characters")
|
||||||
|
# urlsplit cannot distinguish an absent query/fragment from an explicitly
|
||||||
|
# empty one. Reject the delimiters themselves so this validator matches
|
||||||
|
# URLComponents in the client and reviewed data cannot fail closed there.
|
||||||
|
if "?" in value or "#" in value:
|
||||||
|
raise ValidationError(f"relay query or fragment is not allowed: {value}")
|
||||||
|
|
||||||
|
candidate = value if "://" in value else f"wss://{value}"
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(candidate)
|
||||||
|
port = parsed.port
|
||||||
|
except ValueError as error:
|
||||||
|
raise ValidationError(f"invalid relay URL: {value}") from error
|
||||||
|
|
||||||
|
if parsed.scheme.lower() not in {"wss", "https"}:
|
||||||
|
raise ValidationError(f"relay must use wss/https or a bare hostname: {value}")
|
||||||
|
if parsed.username is not None or parsed.password is not None:
|
||||||
|
raise ValidationError(f"relay credentials are not allowed: {value}")
|
||||||
|
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||||
|
raise ValidationError(f"relay path, query, or fragment is not allowed: {value}")
|
||||||
|
|
||||||
|
host = (parsed.hostname or "").lower()
|
||||||
|
if not host or len(host) > 253 or not host.isascii():
|
||||||
|
raise ValidationError(f"relay hostname is missing or non-ASCII: {value}")
|
||||||
|
if host.endswith(".") or host == "localhost" or host.endswith((".localhost", ".local", ".internal")):
|
||||||
|
raise ValidationError(f"local or absolute relay hostname is not allowed: {value}")
|
||||||
|
|
||||||
|
labels = host.split(".")
|
||||||
|
if len(labels) < 2 or all(label.isdigit() for label in labels):
|
||||||
|
raise ValidationError(f"relay must use a public DNS hostname: {value}")
|
||||||
|
for label in labels:
|
||||||
|
if not 1 <= len(label) <= 63:
|
||||||
|
raise ValidationError(f"invalid DNS label length: {value}")
|
||||||
|
if label[0] == "-" or label[-1] == "-":
|
||||||
|
raise ValidationError(f"DNS labels cannot start or end with '-': {value}")
|
||||||
|
if any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label):
|
||||||
|
raise ValidationError(f"invalid DNS hostname character: {value}")
|
||||||
|
|
||||||
|
if port is not None and not 1 <= port <= 65_535:
|
||||||
|
raise ValidationError(f"invalid relay port: {value}")
|
||||||
|
if port in {None, 443}:
|
||||||
|
return host
|
||||||
|
return f"{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _validated_dataset(
|
||||||
|
data: bytes,
|
||||||
|
*,
|
||||||
|
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||||
|
maximum_bytes: int = MAX_BYTES,
|
||||||
|
maximum_rows: int = MAX_ROWS,
|
||||||
|
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||||
|
) -> _ValidatedDataset:
|
||||||
|
if not data or len(data) > maximum_bytes:
|
||||||
|
raise ValidationError(f"CSV must contain 1..{maximum_bytes} bytes")
|
||||||
|
|
||||||
|
try:
|
||||||
|
text = data.decode("utf-8")
|
||||||
|
except UnicodeDecodeError as error:
|
||||||
|
raise ValidationError("CSV is not valid UTF-8") from error
|
||||||
|
if text.startswith("\ufeff"):
|
||||||
|
raise ValidationError("UTF-8 BOM is not allowed")
|
||||||
|
if _has_disallowed_control(text):
|
||||||
|
raise ValidationError("CSV contains disallowed control characters")
|
||||||
|
# Runtime intentionally implements the fixed three-field schema without
|
||||||
|
# general CSV quoting. Reject quoted variants here so reviewed workflow
|
||||||
|
# output and client-side validation cannot disagree.
|
||||||
|
if '"' in text:
|
||||||
|
raise ValidationError("quoted CSV fields are not allowed")
|
||||||
|
|
||||||
|
reader = csv.reader(io.StringIO(text, newline=""), strict=True)
|
||||||
|
try:
|
||||||
|
header = next(reader)
|
||||||
|
except (StopIteration, csv.Error) as error:
|
||||||
|
raise ValidationError("CSV header is missing") from error
|
||||||
|
normalized_header = tuple(field.strip().lower() for field in header)
|
||||||
|
if normalized_header != EXPECTED_HEADER:
|
||||||
|
raise ValidationError(f"unexpected CSV header: {header!r}")
|
||||||
|
|
||||||
|
data_rows = 0
|
||||||
|
relays: dict[str, tuple[float, float]] = {}
|
||||||
|
try:
|
||||||
|
for row in reader:
|
||||||
|
if not row or all(not field.strip() for field in row):
|
||||||
|
continue
|
||||||
|
data_rows += 1
|
||||||
|
if data_rows > maximum_rows:
|
||||||
|
raise ValidationError(f"CSV exceeds {maximum_rows} data rows")
|
||||||
|
if len(row) != 3:
|
||||||
|
raise ValidationError(f"row {reader.line_num} must contain exactly 3 columns")
|
||||||
|
|
||||||
|
address = normalize_relay_address(row[0])
|
||||||
|
latitude_text = row[1].strip()
|
||||||
|
longitude_text = row[2].strip()
|
||||||
|
if not ASCII_DECIMAL_PATTERN.fullmatch(latitude_text) or not ASCII_DECIMAL_PATTERN.fullmatch(longitude_text):
|
||||||
|
raise ValidationError(
|
||||||
|
f"row {reader.line_num} coordinates must be ASCII decimal numbers"
|
||||||
|
)
|
||||||
|
latitude = float(latitude_text)
|
||||||
|
longitude = float(longitude_text)
|
||||||
|
if not math.isfinite(latitude) or not -90 <= latitude <= 90:
|
||||||
|
raise ValidationError(f"row {reader.line_num} latitude is out of range")
|
||||||
|
if not math.isfinite(longitude) or not -180 <= longitude <= 180:
|
||||||
|
raise ValidationError(f"row {reader.line_num} longitude is out of range")
|
||||||
|
|
||||||
|
coordinates = (latitude, longitude)
|
||||||
|
previous = relays.get(address)
|
||||||
|
if previous is not None and previous != coordinates:
|
||||||
|
raise ValidationError(f"relay {address} has conflicting coordinates")
|
||||||
|
relays[address] = coordinates
|
||||||
|
if len(relays) > maximum_unique_relays:
|
||||||
|
raise ValidationError(f"CSV exceeds {maximum_unique_relays} unique relays")
|
||||||
|
except csv.Error as error:
|
||||||
|
raise ValidationError(f"malformed CSV near line {reader.line_num}") from error
|
||||||
|
|
||||||
|
if len(relays) < minimum_unique_relays:
|
||||||
|
raise ValidationError(
|
||||||
|
f"CSV has {len(relays)} unique relays; minimum is {minimum_unique_relays}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return _ValidatedDataset(
|
||||||
|
summary=ValidationSummary(
|
||||||
|
data_rows=data_rows,
|
||||||
|
unique_relays=len(relays),
|
||||||
|
sha256=hashlib.sha256(data).hexdigest(),
|
||||||
|
),
|
||||||
|
entries=frozenset(
|
||||||
|
(address, coordinates[0], coordinates[1])
|
||||||
|
for address, coordinates in relays.items()
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_bytes(
|
||||||
|
data: bytes,
|
||||||
|
*,
|
||||||
|
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||||
|
maximum_bytes: int = MAX_BYTES,
|
||||||
|
maximum_rows: int = MAX_ROWS,
|
||||||
|
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||||
|
) -> ValidationSummary:
|
||||||
|
return _validated_dataset(
|
||||||
|
data,
|
||||||
|
minimum_unique_relays=minimum_unique_relays,
|
||||||
|
maximum_bytes=maximum_bytes,
|
||||||
|
maximum_rows=maximum_rows,
|
||||||
|
maximum_unique_relays=maximum_unique_relays,
|
||||||
|
).summary
|
||||||
|
|
||||||
|
|
||||||
|
def validate_update(candidate: bytes, baseline: bytes) -> ValidationSummary:
|
||||||
|
baseline_dataset = _validated_dataset(baseline, minimum_unique_relays=1)
|
||||||
|
candidate_dataset = _validated_dataset(candidate)
|
||||||
|
baseline_summary = baseline_dataset.summary
|
||||||
|
candidate_summary = candidate_dataset.summary
|
||||||
|
|
||||||
|
minimum_from_baseline = math.ceil(
|
||||||
|
baseline_summary.unique_relays * MIN_BASELINE_FRACTION
|
||||||
|
)
|
||||||
|
maximum_from_baseline = math.floor(
|
||||||
|
baseline_summary.unique_relays * MAX_BASELINE_MULTIPLIER
|
||||||
|
)
|
||||||
|
if candidate_summary.unique_relays < minimum_from_baseline:
|
||||||
|
raise ValidationError(
|
||||||
|
"candidate loses more than half of the baseline's unique relays "
|
||||||
|
f"({candidate_summary.unique_relays} < {minimum_from_baseline})"
|
||||||
|
)
|
||||||
|
if candidate_summary.unique_relays > maximum_from_baseline:
|
||||||
|
raise ValidationError(
|
||||||
|
"candidate more than doubles the baseline's unique relays "
|
||||||
|
f"({candidate_summary.unique_relays} > {maximum_from_baseline})"
|
||||||
|
)
|
||||||
|
|
||||||
|
retained_entries = len(baseline_dataset.entries & candidate_dataset.entries)
|
||||||
|
if retained_entries < minimum_from_baseline:
|
||||||
|
raise ValidationError(
|
||||||
|
"candidate retains fewer than half of the baseline's exact relay-coordinate entries "
|
||||||
|
f"({retained_entries} < {minimum_from_baseline})"
|
||||||
|
)
|
||||||
|
return candidate_summary
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--input", required=True, type=Path)
|
||||||
|
parser.add_argument("--baseline", required=True, type=Path)
|
||||||
|
parser.add_argument("--output", required=True, type=Path)
|
||||||
|
parser.add_argument("--github-output", type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
try:
|
||||||
|
candidate = args.input.read_bytes()
|
||||||
|
baseline = args.baseline.read_bytes()
|
||||||
|
summary = validate_update(candidate, baseline)
|
||||||
|
args.output.write_bytes(candidate)
|
||||||
|
if args.github_output is not None:
|
||||||
|
with args.github_output.open("a", encoding="utf-8") as output:
|
||||||
|
output.write(f"data_rows={summary.data_rows}\n")
|
||||||
|
output.write(f"unique_relays={summary.unique_relays}\n")
|
||||||
|
output.write(f"sha256={summary.sha256}\n")
|
||||||
|
except (OSError, ValidationError) as error:
|
||||||
|
print(f"georelay validation failed: {error}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"validated {summary.unique_relays} unique relays across "
|
||||||
|
f"{summary.data_rows} rows (sha256 {summary.sha256})"
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in New Issue
Block a user