mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-25 07:25:19 +00:00
Route georelay updates through review
This commit is contained in:
jack
@@ -0,0 +1,61 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW_PATH = REPOSITORY_ROOT / ".github/workflows/fetch_georelays.yml"
|
||||
|
||||
|
||||
class FetchGeoRelaysWorkflowTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_write_capable_checkout_action_is_immutable(self) -> None:
|
||||
checkout = re.search(r"uses: actions/checkout@([0-9a-f]+)", self.workflow)
|
||||
self.assertIsNotNone(checkout)
|
||||
self.assertRegex(checkout.group(1), r"^[0-9a-f]{40}$")
|
||||
self.assertIn("persist-credentials: false", self.workflow)
|
||||
|
||||
def test_pr_failure_has_single_issue_fallback_with_review_metadata(self) -> None:
|
||||
required_fragments = [
|
||||
"issues: write",
|
||||
"TRACKING_ISSUE_TITLE: GeoRelay update awaiting pull request",
|
||||
"gh pr create",
|
||||
"gh issue create",
|
||||
"gh issue edit",
|
||||
"compare/main...${UPDATE_BRANCH}?expand=1",
|
||||
"Upstream commit: $SOURCE_COMMIT",
|
||||
"Data rows: $DATA_ROWS",
|
||||
"Unique normalized relays: $UNIQUE_RELAYS",
|
||||
"SHA-256: $DATA_SHA256",
|
||||
'[[ -n "$issue_url" ]]',
|
||||
]
|
||||
for fragment in required_fragments:
|
||||
with self.subTest(fragment=fragment):
|
||||
self.assertIn(fragment, self.workflow)
|
||||
|
||||
confirmed = self.workflow.index('[[ -n "$issue_url" ]]')
|
||||
success_summary = self.workflow.index(
|
||||
"Published GeoRelay tracking issue fallback: $issue_url"
|
||||
)
|
||||
self.assertLess(confirmed, success_summary)
|
||||
|
||||
def test_obsolete_review_state_is_cleaned_without_pushing_main(self) -> None:
|
||||
self.assertIn("gh pr close", self.workflow)
|
||||
self.assertIn("gh issue close", self.workflow)
|
||||
self.assertIn('git push origin --delete "$UPDATE_BRANCH"', self.workflow)
|
||||
self.assertIn('git switch -C "$UPDATE_BRANCH"', self.workflow)
|
||||
self.assertNotIn("git push origin main", self.workflow)
|
||||
self.assertNotIn("git push --force origin main", self.workflow)
|
||||
|
||||
def test_workflow_runs_all_validator_tests(self) -> None:
|
||||
self.assertIn(
|
||||
'python3 -m unittest discover -s scripts/tests -p "test_*.py" -v',
|
||||
self.workflow,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,186 @@
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import validate_georelays as validator
|
||||
|
||||
|
||||
def csv_bytes(rows: list[str]) -> bytes:
|
||||
return ("Relay URL,Latitude,Longitude\n" + "\n".join(rows) + "\n").encode()
|
||||
|
||||
|
||||
class ValidateGeoRelaysTests(unittest.TestCase):
|
||||
def test_validates_and_deduplicates_secure_relay_addresses(self) -> None:
|
||||
data = csv_bytes(
|
||||
[
|
||||
"relay.example.com,10,20",
|
||||
"wss://relay.example.com:443/,10,20",
|
||||
"https://second.example.org,11,21",
|
||||
]
|
||||
)
|
||||
|
||||
summary = validator.validate_bytes(data, minimum_unique_relays=2)
|
||||
|
||||
self.assertEqual(summary.data_rows, 3)
|
||||
self.assertEqual(summary.unique_relays, 2)
|
||||
|
||||
def test_rejects_insecure_or_non_host_relay_urls(self) -> None:
|
||||
bad_addresses = [
|
||||
"http://relay.example.com",
|
||||
"ws://relay.example.com",
|
||||
"wss://user@relay.example.com",
|
||||
"wss://relay.example.com/path",
|
||||
"wss://relay.example.com?",
|
||||
"wss://relay.example.com#",
|
||||
"relay.example.com:0",
|
||||
"relay.example.com:99999",
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"relay_example.com",
|
||||
"relay\u202e.example.com",
|
||||
]
|
||||
|
||||
for address in bad_addresses:
|
||||
with self.subTest(address=address):
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes([f"{address},10,20"]),
|
||||
minimum_unique_relays=1,
|
||||
)
|
||||
|
||||
def test_rejects_malformed_rows_and_unsafe_coordinates(self) -> None:
|
||||
bad_rows = [
|
||||
"relay.example.com,10",
|
||||
"relay.example.com,NaN,20",
|
||||
"relay.example.com,1_0,20",
|
||||
"relay.example.com,\u0661\u0660,20",
|
||||
"relay.example.com,\uff11\uff10,20",
|
||||
"relay.example.com,91,20",
|
||||
"relay.example.com,10,-181",
|
||||
"relay.example.com,10,20,extra",
|
||||
'"relay.example.com",10,20',
|
||||
]
|
||||
|
||||
for row in bad_rows:
|
||||
with self.subTest(row=row):
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(csv_bytes([row]), minimum_unique_relays=1)
|
||||
|
||||
def test_accepts_ascii_coordinate_forms_supported_by_swift_double(self) -> None:
|
||||
summary = validator.validate_bytes(
|
||||
csv_bytes(
|
||||
[
|
||||
"one.example.com,+1,-.5",
|
||||
"two.example.com,1.e1,2E+1",
|
||||
"three.example.com,01,20.",
|
||||
]
|
||||
),
|
||||
minimum_unique_relays=3,
|
||||
)
|
||||
|
||||
self.assertEqual(summary.unique_relays, 3)
|
||||
|
||||
def test_rejects_conflicts_limits_and_large_baseline_deltas(self) -> None:
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes(["relay.example.com,10,20", "relay.example.com,11,21"]),
|
||||
minimum_unique_relays=1,
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(b"x" * 20, maximum_bytes=10, minimum_unique_relays=1)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_bytes(
|
||||
csv_bytes(["one.example.com,1,1", "two.example.com,2,2"]),
|
||||
minimum_unique_relays=3,
|
||||
)
|
||||
|
||||
baseline = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(120)]
|
||||
)
|
||||
shrunken = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(59)]
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_update(shrunken, baseline)
|
||||
|
||||
smaller_baseline = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||
)
|
||||
expanded = csv_bytes(
|
||||
[f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(121)]
|
||||
)
|
||||
with self.assertRaises(validator.ValidationError):
|
||||
validator.validate_update(expanded, smaller_baseline)
|
||||
|
||||
def test_update_requires_exact_normalized_baseline_entry_overlap(self) -> None:
|
||||
baseline_rows = [
|
||||
f"relay-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
baseline = csv_bytes(baseline_rows)
|
||||
disjoint = csv_bytes(
|
||||
[
|
||||
f"attacker-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
)
|
||||
rewritten_coordinates = csv_bytes(
|
||||
[
|
||||
f"relay-{index}.example.com,{(index % 80) + 0.5},{index % 170}"
|
||||
for index in range(60)
|
||||
]
|
||||
)
|
||||
|
||||
for candidate in (disjoint, rewritten_coordinates):
|
||||
with self.subTest(candidate=candidate[:80]):
|
||||
with self.assertRaisesRegex(
|
||||
validator.ValidationError,
|
||||
"exact relay-coordinate entries",
|
||||
):
|
||||
validator.validate_update(candidate, baseline)
|
||||
|
||||
half_retained = csv_bytes(
|
||||
[
|
||||
f"wss://relay-{index}.example.com:443/,{index % 80},{index % 170}"
|
||||
for index in range(30)
|
||||
]
|
||||
+ [
|
||||
f"replacement-{index}.example.com,{index % 80},{index % 170}"
|
||||
for index in range(30)
|
||||
]
|
||||
)
|
||||
summary = validator.validate_update(half_retained, baseline)
|
||||
self.assertEqual(summary.unique_relays, 60)
|
||||
|
||||
def test_cli_copies_only_validated_data_and_emits_review_metadata(self) -> None:
|
||||
rows = [f"relay-{index}.example.com,{index % 80},{index % 170}" for index in range(60)]
|
||||
data = csv_bytes(rows)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
candidate = root / "candidate.csv"
|
||||
baseline = root / "baseline.csv"
|
||||
output = root / "output.csv"
|
||||
github_output = root / "github-output.txt"
|
||||
candidate.write_bytes(data)
|
||||
baseline.write_bytes(data)
|
||||
|
||||
result = validator.main(
|
||||
[
|
||||
"--input", str(candidate),
|
||||
"--baseline", str(baseline),
|
||||
"--output", str(output),
|
||||
"--github-output", str(github_output),
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertEqual(output.read_bytes(), data)
|
||||
metadata = github_output.read_text()
|
||||
self.assertIn("unique_relays=60", metadata)
|
||||
self.assertIn("sha256=", metadata)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict validator for the reviewed georelay CSV update workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unicodedata
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
MAX_BYTES = 512 * 1024
|
||||
MAX_ROWS = 5_000
|
||||
MAX_UNIQUE_RELAYS = 5_000
|
||||
MIN_UNIQUE_RELAYS = 50
|
||||
MIN_BASELINE_FRACTION = 0.5
|
||||
MAX_BASELINE_MULTIPLIER = 2.0
|
||||
EXPECTED_HEADER = ("relay url", "latitude", "longitude")
|
||||
ASCII_DECIMAL_PATTERN = re.compile(
|
||||
r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z"
|
||||
)
|
||||
|
||||
|
||||
class ValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidationSummary:
|
||||
data_rows: int
|
||||
unique_relays: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ValidatedDataset:
|
||||
summary: ValidationSummary
|
||||
entries: frozenset[tuple[str, float, float]]
|
||||
|
||||
|
||||
def _has_disallowed_control(value: str) -> bool:
|
||||
return any(
|
||||
unicodedata.category(character) in {"Cc", "Cf"}
|
||||
and character not in {"\r", "\n", "\t"}
|
||||
for character in value
|
||||
)
|
||||
|
||||
|
||||
def normalize_relay_address(raw_value: str) -> str:
|
||||
value = raw_value.strip()
|
||||
if not value or _has_disallowed_control(value):
|
||||
raise ValidationError("relay address is empty or contains control characters")
|
||||
# urlsplit cannot distinguish an absent query/fragment from an explicitly
|
||||
# empty one. Reject the delimiters themselves so this validator matches
|
||||
# URLComponents in the client and reviewed data cannot fail closed there.
|
||||
if "?" in value or "#" in value:
|
||||
raise ValidationError(f"relay query or fragment is not allowed: {value}")
|
||||
|
||||
candidate = value if "://" in value else f"wss://{value}"
|
||||
try:
|
||||
parsed = urlsplit(candidate)
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValidationError(f"invalid relay URL: {value}") from error
|
||||
|
||||
if parsed.scheme.lower() not in {"wss", "https"}:
|
||||
raise ValidationError(f"relay must use wss/https or a bare hostname: {value}")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValidationError(f"relay credentials are not allowed: {value}")
|
||||
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||
raise ValidationError(f"relay path, query, or fragment is not allowed: {value}")
|
||||
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host or len(host) > 253 or not host.isascii():
|
||||
raise ValidationError(f"relay hostname is missing or non-ASCII: {value}")
|
||||
if host.endswith(".") or host == "localhost" or host.endswith((".localhost", ".local", ".internal")):
|
||||
raise ValidationError(f"local or absolute relay hostname is not allowed: {value}")
|
||||
|
||||
labels = host.split(".")
|
||||
if len(labels) < 2 or all(label.isdigit() for label in labels):
|
||||
raise ValidationError(f"relay must use a public DNS hostname: {value}")
|
||||
for label in labels:
|
||||
if not 1 <= len(label) <= 63:
|
||||
raise ValidationError(f"invalid DNS label length: {value}")
|
||||
if label[0] == "-" or label[-1] == "-":
|
||||
raise ValidationError(f"DNS labels cannot start or end with '-': {value}")
|
||||
if any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label):
|
||||
raise ValidationError(f"invalid DNS hostname character: {value}")
|
||||
|
||||
if port is not None and not 1 <= port <= 65_535:
|
||||
raise ValidationError(f"invalid relay port: {value}")
|
||||
if port in {None, 443}:
|
||||
return host
|
||||
return f"{host}:{port}"
|
||||
|
||||
|
||||
def _validated_dataset(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> _ValidatedDataset:
|
||||
if not data or len(data) > maximum_bytes:
|
||||
raise ValidationError(f"CSV must contain 1..{maximum_bytes} bytes")
|
||||
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValidationError("CSV is not valid UTF-8") from error
|
||||
if text.startswith("\ufeff"):
|
||||
raise ValidationError("UTF-8 BOM is not allowed")
|
||||
if _has_disallowed_control(text):
|
||||
raise ValidationError("CSV contains disallowed control characters")
|
||||
# Runtime intentionally implements the fixed three-field schema without
|
||||
# general CSV quoting. Reject quoted variants here so reviewed workflow
|
||||
# output and client-side validation cannot disagree.
|
||||
if '"' in text:
|
||||
raise ValidationError("quoted CSV fields are not allowed")
|
||||
|
||||
reader = csv.reader(io.StringIO(text, newline=""), strict=True)
|
||||
try:
|
||||
header = next(reader)
|
||||
except (StopIteration, csv.Error) as error:
|
||||
raise ValidationError("CSV header is missing") from error
|
||||
normalized_header = tuple(field.strip().lower() for field in header)
|
||||
if normalized_header != EXPECTED_HEADER:
|
||||
raise ValidationError(f"unexpected CSV header: {header!r}")
|
||||
|
||||
data_rows = 0
|
||||
relays: dict[str, tuple[float, float]] = {}
|
||||
try:
|
||||
for row in reader:
|
||||
if not row or all(not field.strip() for field in row):
|
||||
continue
|
||||
data_rows += 1
|
||||
if data_rows > maximum_rows:
|
||||
raise ValidationError(f"CSV exceeds {maximum_rows} data rows")
|
||||
if len(row) != 3:
|
||||
raise ValidationError(f"row {reader.line_num} must contain exactly 3 columns")
|
||||
|
||||
address = normalize_relay_address(row[0])
|
||||
latitude_text = row[1].strip()
|
||||
longitude_text = row[2].strip()
|
||||
if not ASCII_DECIMAL_PATTERN.fullmatch(latitude_text) or not ASCII_DECIMAL_PATTERN.fullmatch(longitude_text):
|
||||
raise ValidationError(
|
||||
f"row {reader.line_num} coordinates must be ASCII decimal numbers"
|
||||
)
|
||||
latitude = float(latitude_text)
|
||||
longitude = float(longitude_text)
|
||||
if not math.isfinite(latitude) or not -90 <= latitude <= 90:
|
||||
raise ValidationError(f"row {reader.line_num} latitude is out of range")
|
||||
if not math.isfinite(longitude) or not -180 <= longitude <= 180:
|
||||
raise ValidationError(f"row {reader.line_num} longitude is out of range")
|
||||
|
||||
coordinates = (latitude, longitude)
|
||||
previous = relays.get(address)
|
||||
if previous is not None and previous != coordinates:
|
||||
raise ValidationError(f"relay {address} has conflicting coordinates")
|
||||
relays[address] = coordinates
|
||||
if len(relays) > maximum_unique_relays:
|
||||
raise ValidationError(f"CSV exceeds {maximum_unique_relays} unique relays")
|
||||
except csv.Error as error:
|
||||
raise ValidationError(f"malformed CSV near line {reader.line_num}") from error
|
||||
|
||||
if len(relays) < minimum_unique_relays:
|
||||
raise ValidationError(
|
||||
f"CSV has {len(relays)} unique relays; minimum is {minimum_unique_relays}"
|
||||
)
|
||||
|
||||
return _ValidatedDataset(
|
||||
summary=ValidationSummary(
|
||||
data_rows=data_rows,
|
||||
unique_relays=len(relays),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
),
|
||||
entries=frozenset(
|
||||
(address, coordinates[0], coordinates[1])
|
||||
for address, coordinates in relays.items()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def validate_bytes(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> ValidationSummary:
|
||||
return _validated_dataset(
|
||||
data,
|
||||
minimum_unique_relays=minimum_unique_relays,
|
||||
maximum_bytes=maximum_bytes,
|
||||
maximum_rows=maximum_rows,
|
||||
maximum_unique_relays=maximum_unique_relays,
|
||||
).summary
|
||||
|
||||
|
||||
def validate_update(candidate: bytes, baseline: bytes) -> ValidationSummary:
|
||||
baseline_dataset = _validated_dataset(baseline, minimum_unique_relays=1)
|
||||
candidate_dataset = _validated_dataset(candidate)
|
||||
baseline_summary = baseline_dataset.summary
|
||||
candidate_summary = candidate_dataset.summary
|
||||
|
||||
minimum_from_baseline = math.ceil(
|
||||
baseline_summary.unique_relays * MIN_BASELINE_FRACTION
|
||||
)
|
||||
maximum_from_baseline = math.floor(
|
||||
baseline_summary.unique_relays * MAX_BASELINE_MULTIPLIER
|
||||
)
|
||||
if candidate_summary.unique_relays < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate loses more than half of the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} < {minimum_from_baseline})"
|
||||
)
|
||||
if candidate_summary.unique_relays > maximum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate more than doubles the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} > {maximum_from_baseline})"
|
||||
)
|
||||
|
||||
retained_entries = len(baseline_dataset.entries & candidate_dataset.entries)
|
||||
if retained_entries < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate retains fewer than half of the baseline's exact relay-coordinate entries "
|
||||
f"({retained_entries} < {minimum_from_baseline})"
|
||||
)
|
||||
return candidate_summary
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", required=True, type=Path)
|
||||
parser.add_argument("--baseline", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--github-output", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
candidate = args.input.read_bytes()
|
||||
baseline = args.baseline.read_bytes()
|
||||
summary = validate_update(candidate, baseline)
|
||||
args.output.write_bytes(candidate)
|
||||
if args.github_output is not None:
|
||||
with args.github_output.open("a", encoding="utf-8") as output:
|
||||
output.write(f"data_rows={summary.data_rows}\n")
|
||||
output.write(f"unique_relays={summary.unique_relays}\n")
|
||||
output.write(f"sha256={summary.sha256}\n")
|
||||
except (OSError, ValidationError) as error:
|
||||
print(f"georelay validation failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"validated {summary.unique_relays} unique relays across "
|
||||
f"{summary.data_rows} rows (sha256 {summary.sha256})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user