feat: design cypherpunk IRC georelays dashboard for github.io

scripts/geolocate_all_relays.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Geolocate All Discovered Relays
+
+This script reads:
+1. nostr_relays.csv - The already geolocated BitChat-capable relays
+2. relay_discovery_results.json - The list of ALL functioning relays discovered
+
+It geolocates the remaining non-BitChat relays and outputs a unified CSV:
+'all_relays_geo.csv' with columns: Relay URL,Latitude,Longitude,IsBitChat (1 or 0).
+This avoids redundant geolocating of the BitChat relays.
+"""
+
+import os
+import sys
+import csv
+import json
+import asyncio
+import re
+
+# Add root folder to path to import relays_geo_lookup
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+try:
+    from relays_geo_lookup import GeoIPDatabase, resolve_and_locate, clean_url, DB_FILENAME
+except ImportError as e:
+    print(f"Error importing modules: {e}")
+    sys.exit(1)
+
+def strip_protocol(url):
+    """Standardizes URLs by removing ws:// or wss:// prefix."""
+    return re.sub(r'^wss?://', '', url).strip()
+
+async def main():
+    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    os.chdir(root_dir)
+
+    bitchat_csv_path = "nostr_relays.csv"
+    results_json_path = "relay_discovery_results.json"
+    output_csv_path = "all_relays_geo.csv"
+
+    if not os.path.exists(bitchat_csv_path):
+        print(f"Error: {bitchat_csv_path} does not exist. Run the pipeline first.")
+        sys.exit(1)
+    if not os.path.exists(results_json_path):
+        print(f"Error: {results_json_path} does not exist.")
+        sys.exit(1)
+
+    # 1. Read already geolocated BitChat relays
+    bitchat_relays = {}
+    with open(bitchat_csv_path, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        header = next(reader, None)
+        for row in reader:
+            if len(row) >= 3:
+                url, lat, lon = row[0].strip(), row[1].strip(), row[2].strip()
+                # Store mapped by both raw URL and stripped protocol URL for matching robustness
+                bitchat_relays[strip_protocol(url)] = (url, lat, lon)
+
+    print(f"Loaded {len(bitchat_relays)} geolocated BitChat relays.")
+
+    # 2. Read all functioning relays from discovery JSON
+    with open(results_json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+        all_functioning = data.get("functioning_relays", [])
+
+    print(f"Loaded {len(all_functioning)} total functioning relays from discovery results.")
+
+    # 3. Filter out those that are NOT geolocated BitChat relays
+    non_bitchat_to_locate = []
+    for raw_url in all_functioning:
+        stripped = strip_protocol(raw_url)
+        if stripped not in bitchat_relays:
+            non_bitchat_to_locate.append(raw_url)
+
+    print(f"Identified {len(non_bitchat_to_locate)} non-BitChat relays requiring geolocation.")
+
+    # 4. Geolocate the non-BitChat relays asynchronously
+    success_records = []
+    if non_bitchat_to_locate:
+        db = GeoIPDatabase(DB_FILENAME)
+        db.ensure_db_exists()
+        db.load()
+
+        print(f"Geolocating {len(non_bitchat_to_locate)} relays...")
+        tasks = [resolve_and_locate(url, url, db) for url in non_bitchat_to_locate]
+        results = await asyncio.gather(*tasks)
+
+        for res in results:
+            if res:
+                url, lat, lon = res
+                success_records.append((url, lat, lon))
+        print(f"Successfully geolocated {len(success_records)}/{len(non_bitchat_to_locate)} non-BitChat relays.")
+
+    # 5. Write unified CSV output
+    total_written = 0
+    with open(output_csv_path, 'w', newline='', encoding='utf-8') as f:
+        writer = csv.writer(f)
+        writer.writerow(["Relay URL", "Latitude", "Longitude", "IsBitChat"])
+
+        # Write BitChat relays (IsBitChat = 1)
+        for stripped_url, (url, lat, lon) in bitchat_relays.items():
+            # Keep protocol-stripped for consistency if that's how it is in the CSV
+            writer.writerow([url, lat, lon, "1"])
+            total_written += 1
+
+        # Write geolocated non-BitChat relays (IsBitChat = 0)
+        for url, lat, lon in success_records:
+            # Strip protocol to match the other CSV format
+            clean_r = strip_protocol(url)
+            writer.writerow([clean_r, lat, lon, "0"])
+            total_written += 1
+
+    print(f"Unified database written to {output_csv_path} (Total of {total_written} geolocated relays).")
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nInterrupted.")
+        sys.exit(1)

scripts/track_relay_counts.py

@@ -220,6 +220,35 @@ def main():
         'assets/total_relay_count_chart.png'
     )
     
+    # Save historical data to JSON for interactive graphs
+    history_json_path = 'assets/relay_history.json'
+    bitchat_serializable = []
+    for entry in bitchat_data:
+        bitchat_serializable.append({
+            'date': entry['date'].strftime('%Y-%m-%d'),
+            'count': entry['count'],
+            'commit': entry['commit']
+        })
+    bitchat_serializable.sort(key=lambda x: x['date'])
+
+    total_serializable = []
+    for entry in total_data:
+        total_serializable.append({
+            'date': entry['date'].strftime('%Y-%m-%d'),
+            'count': entry['count'],
+            'commit': entry['commit']
+        })
+    total_serializable.sort(key=lambda x: x['date'])
+
+    history_payload = {
+        'bitchat': bitchat_serializable,
+        'total': total_serializable
+    }
+    
+    with open(history_json_path, 'w') as f:
+        json.dump(history_payload, f, indent=2)
+    print(f"Exported interactive charts history to {history_json_path}")
+
     # Print summary for log
     if bitchat_stats:
         print(f"Generated BitChat relay chart with {bitchat_stats[0]} data points")