mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-07-25 08:05:20 +00:00
Route georelay updates through review
This commit is contained in:
jack
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict validator for the reviewed georelay CSV update workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unicodedata
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
MAX_BYTES = 512 * 1024
|
||||
MAX_ROWS = 5_000
|
||||
MAX_UNIQUE_RELAYS = 5_000
|
||||
MIN_UNIQUE_RELAYS = 50
|
||||
MIN_BASELINE_FRACTION = 0.5
|
||||
MAX_BASELINE_MULTIPLIER = 2.0
|
||||
EXPECTED_HEADER = ("relay url", "latitude", "longitude")
|
||||
ASCII_DECIMAL_PATTERN = re.compile(
|
||||
r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\Z"
|
||||
)
|
||||
|
||||
|
||||
class ValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidationSummary:
|
||||
data_rows: int
|
||||
unique_relays: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ValidatedDataset:
|
||||
summary: ValidationSummary
|
||||
entries: frozenset[tuple[str, float, float]]
|
||||
|
||||
|
||||
def _has_disallowed_control(value: str) -> bool:
|
||||
return any(
|
||||
unicodedata.category(character) in {"Cc", "Cf"}
|
||||
and character not in {"\r", "\n", "\t"}
|
||||
for character in value
|
||||
)
|
||||
|
||||
|
||||
def normalize_relay_address(raw_value: str) -> str:
|
||||
value = raw_value.strip()
|
||||
if not value or _has_disallowed_control(value):
|
||||
raise ValidationError("relay address is empty or contains control characters")
|
||||
# urlsplit cannot distinguish an absent query/fragment from an explicitly
|
||||
# empty one. Reject the delimiters themselves so this validator matches
|
||||
# URLComponents in the client and reviewed data cannot fail closed there.
|
||||
if "?" in value or "#" in value:
|
||||
raise ValidationError(f"relay query or fragment is not allowed: {value}")
|
||||
|
||||
candidate = value if "://" in value else f"wss://{value}"
|
||||
try:
|
||||
parsed = urlsplit(candidate)
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValidationError(f"invalid relay URL: {value}") from error
|
||||
|
||||
if parsed.scheme.lower() not in {"wss", "https"}:
|
||||
raise ValidationError(f"relay must use wss/https or a bare hostname: {value}")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValidationError(f"relay credentials are not allowed: {value}")
|
||||
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||
raise ValidationError(f"relay path, query, or fragment is not allowed: {value}")
|
||||
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host or len(host) > 253 or not host.isascii():
|
||||
raise ValidationError(f"relay hostname is missing or non-ASCII: {value}")
|
||||
if host.endswith(".") or host == "localhost" or host.endswith((".localhost", ".local", ".internal")):
|
||||
raise ValidationError(f"local or absolute relay hostname is not allowed: {value}")
|
||||
|
||||
labels = host.split(".")
|
||||
if len(labels) < 2 or all(label.isdigit() for label in labels):
|
||||
raise ValidationError(f"relay must use a public DNS hostname: {value}")
|
||||
for label in labels:
|
||||
if not 1 <= len(label) <= 63:
|
||||
raise ValidationError(f"invalid DNS label length: {value}")
|
||||
if label[0] == "-" or label[-1] == "-":
|
||||
raise ValidationError(f"DNS labels cannot start or end with '-': {value}")
|
||||
if any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label):
|
||||
raise ValidationError(f"invalid DNS hostname character: {value}")
|
||||
|
||||
if port is not None and not 1 <= port <= 65_535:
|
||||
raise ValidationError(f"invalid relay port: {value}")
|
||||
if port in {None, 443}:
|
||||
return host
|
||||
return f"{host}:{port}"
|
||||
|
||||
|
||||
def _validated_dataset(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> _ValidatedDataset:
|
||||
if not data or len(data) > maximum_bytes:
|
||||
raise ValidationError(f"CSV must contain 1..{maximum_bytes} bytes")
|
||||
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValidationError("CSV is not valid UTF-8") from error
|
||||
if text.startswith("\ufeff"):
|
||||
raise ValidationError("UTF-8 BOM is not allowed")
|
||||
if _has_disallowed_control(text):
|
||||
raise ValidationError("CSV contains disallowed control characters")
|
||||
# Runtime intentionally implements the fixed three-field schema without
|
||||
# general CSV quoting. Reject quoted variants here so reviewed workflow
|
||||
# output and client-side validation cannot disagree.
|
||||
if '"' in text:
|
||||
raise ValidationError("quoted CSV fields are not allowed")
|
||||
|
||||
reader = csv.reader(io.StringIO(text, newline=""), strict=True)
|
||||
try:
|
||||
header = next(reader)
|
||||
except (StopIteration, csv.Error) as error:
|
||||
raise ValidationError("CSV header is missing") from error
|
||||
normalized_header = tuple(field.strip().lower() for field in header)
|
||||
if normalized_header != EXPECTED_HEADER:
|
||||
raise ValidationError(f"unexpected CSV header: {header!r}")
|
||||
|
||||
data_rows = 0
|
||||
relays: dict[str, tuple[float, float]] = {}
|
||||
try:
|
||||
for row in reader:
|
||||
if not row or all(not field.strip() for field in row):
|
||||
continue
|
||||
data_rows += 1
|
||||
if data_rows > maximum_rows:
|
||||
raise ValidationError(f"CSV exceeds {maximum_rows} data rows")
|
||||
if len(row) != 3:
|
||||
raise ValidationError(f"row {reader.line_num} must contain exactly 3 columns")
|
||||
|
||||
address = normalize_relay_address(row[0])
|
||||
latitude_text = row[1].strip()
|
||||
longitude_text = row[2].strip()
|
||||
if not ASCII_DECIMAL_PATTERN.fullmatch(latitude_text) or not ASCII_DECIMAL_PATTERN.fullmatch(longitude_text):
|
||||
raise ValidationError(
|
||||
f"row {reader.line_num} coordinates must be ASCII decimal numbers"
|
||||
)
|
||||
latitude = float(latitude_text)
|
||||
longitude = float(longitude_text)
|
||||
if not math.isfinite(latitude) or not -90 <= latitude <= 90:
|
||||
raise ValidationError(f"row {reader.line_num} latitude is out of range")
|
||||
if not math.isfinite(longitude) or not -180 <= longitude <= 180:
|
||||
raise ValidationError(f"row {reader.line_num} longitude is out of range")
|
||||
|
||||
coordinates = (latitude, longitude)
|
||||
previous = relays.get(address)
|
||||
if previous is not None and previous != coordinates:
|
||||
raise ValidationError(f"relay {address} has conflicting coordinates")
|
||||
relays[address] = coordinates
|
||||
if len(relays) > maximum_unique_relays:
|
||||
raise ValidationError(f"CSV exceeds {maximum_unique_relays} unique relays")
|
||||
except csv.Error as error:
|
||||
raise ValidationError(f"malformed CSV near line {reader.line_num}") from error
|
||||
|
||||
if len(relays) < minimum_unique_relays:
|
||||
raise ValidationError(
|
||||
f"CSV has {len(relays)} unique relays; minimum is {minimum_unique_relays}"
|
||||
)
|
||||
|
||||
return _ValidatedDataset(
|
||||
summary=ValidationSummary(
|
||||
data_rows=data_rows,
|
||||
unique_relays=len(relays),
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
),
|
||||
entries=frozenset(
|
||||
(address, coordinates[0], coordinates[1])
|
||||
for address, coordinates in relays.items()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def validate_bytes(
|
||||
data: bytes,
|
||||
*,
|
||||
minimum_unique_relays: int = MIN_UNIQUE_RELAYS,
|
||||
maximum_bytes: int = MAX_BYTES,
|
||||
maximum_rows: int = MAX_ROWS,
|
||||
maximum_unique_relays: int = MAX_UNIQUE_RELAYS,
|
||||
) -> ValidationSummary:
|
||||
return _validated_dataset(
|
||||
data,
|
||||
minimum_unique_relays=minimum_unique_relays,
|
||||
maximum_bytes=maximum_bytes,
|
||||
maximum_rows=maximum_rows,
|
||||
maximum_unique_relays=maximum_unique_relays,
|
||||
).summary
|
||||
|
||||
|
||||
def validate_update(candidate: bytes, baseline: bytes) -> ValidationSummary:
|
||||
baseline_dataset = _validated_dataset(baseline, minimum_unique_relays=1)
|
||||
candidate_dataset = _validated_dataset(candidate)
|
||||
baseline_summary = baseline_dataset.summary
|
||||
candidate_summary = candidate_dataset.summary
|
||||
|
||||
minimum_from_baseline = math.ceil(
|
||||
baseline_summary.unique_relays * MIN_BASELINE_FRACTION
|
||||
)
|
||||
maximum_from_baseline = math.floor(
|
||||
baseline_summary.unique_relays * MAX_BASELINE_MULTIPLIER
|
||||
)
|
||||
if candidate_summary.unique_relays < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate loses more than half of the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} < {minimum_from_baseline})"
|
||||
)
|
||||
if candidate_summary.unique_relays > maximum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate more than doubles the baseline's unique relays "
|
||||
f"({candidate_summary.unique_relays} > {maximum_from_baseline})"
|
||||
)
|
||||
|
||||
retained_entries = len(baseline_dataset.entries & candidate_dataset.entries)
|
||||
if retained_entries < minimum_from_baseline:
|
||||
raise ValidationError(
|
||||
"candidate retains fewer than half of the baseline's exact relay-coordinate entries "
|
||||
f"({retained_entries} < {minimum_from_baseline})"
|
||||
)
|
||||
return candidate_summary
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", required=True, type=Path)
|
||||
parser.add_argument("--baseline", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--github-output", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
candidate = args.input.read_bytes()
|
||||
baseline = args.baseline.read_bytes()
|
||||
summary = validate_update(candidate, baseline)
|
||||
args.output.write_bytes(candidate)
|
||||
if args.github_output is not None:
|
||||
with args.github_output.open("a", encoding="utf-8") as output:
|
||||
output.write(f"data_rows={summary.data_rows}\n")
|
||||
output.write(f"unique_relays={summary.unique_relays}\n")
|
||||
output.write(f"sha256={summary.sha256}\n")
|
||||
except (OSError, ValidationError) as error:
|
||||
print(f"georelay validation failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"validated {summary.unique_relays} unique relays across "
|
||||
f"{summary.data_rows} rows (sha256 {summary.sha256})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user