diff --git a/.github/workflows/relay-count-tracker.yml b/.github/workflows/relay-count-tracker.yml new file mode 100644 index 0000000..52a7e6f --- /dev/null +++ b/.github/workflows/relay-count-tracker.yml @@ -0,0 +1,64 @@ +name: Track Relay Count Changes + +on: + # Run after the "Update Relay Data" workflow completes + workflow_run: + workflows: ["Update Relay Data"] + types: + - completed + # Also allow manual triggering + workflow_dispatch: + +permissions: + contents: write + +jobs: + track-relay-count: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 100 # Need deeper history to check previous commits (70+ days) + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + # Check if requirements.txt includes matplotlib, pandas, numpy + if ! grep -q "matplotlib\|pandas\|numpy" requirements.txt; then + # If not, install them directly + pip install matplotlib pandas numpy + else + # Otherwise use the requirements file + pip install -r requirements.txt + fi + + - name: Track relay count changes + run: | + mkdir -p assets + + # Execute the script + python scripts/track_relay_counts.py + + - name: Check for changes + id: git-check + run: | + git add assets/bitchat_relay_count_chart.png assets/total_relay_count_chart.png + git diff --staged --exit-code || echo "changes=true" >> $GITHUB_OUTPUT + + - name: Commit and push changes + if: steps.git-check.outputs.changes == 'true' + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git commit -m "Update relay count charts - $(date -u)" + git push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b281304 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual Environment +venv/ +env/ +ENV/ + +# IDE files +.idea/ +.vscode/ diff --git a/PROJECT_STRUCTURE.md b/PROJECT_STRUCTURE.md new file mode 100644 index 0000000..ff86b1b --- /dev/null +++ b/PROJECT_STRUCTURE.md @@ -0,0 +1,23 @@ +# Project Organization + +This project is organized as follows: + +## Core Files +- `nostr_relay_discovery.py` - Python script for discovering functioning Nostr relays +- `filter_bitchat_relays.sh` - Shell script to filter relays for BitChat capability +- `relays_geo_lookup.sh` - Shell script to geolocate relay servers +- `nostr_relays.csv` - The main output file with relay URLs and geolocation data + +## Directories +- `/assets` - Contains generated images and other static resources + - `relay_count_chart.png` - Chart showing relay count history +- `/scripts` - Utility scripts for analysis and visualization + - `track_relay_counts.py` - Script for analyzing relay count changes +- `/.github/workflows` - GitHub Actions workflow definitions + - `update-relay-data.yml` - Workflow that updates relay data daily + - `relay-count-tracker.yml` - Workflow that tracks relay count changes + +## GitHub Workflows +The project uses two automated workflows: +1. `update-relay-data.yml` - Runs daily at 6:00 AM UTC to update the relay data +2. `relay-count-tracker.yml` - Runs after the update workflow to track and visualize changes diff --git a/README.md b/README.md index d375965..83255a1 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,23 @@ To change the schedule, seed relay, or enable BitChat filtering in CI, edit the --- +## Relay Count History + +These charts show the number of Nostr relay entries in our dataset over time: + +### BitChat-Compatible Relays + +The chart below shows relays that support BitChat events (kind 20000): + +![BitChat-Compatible Relay Count Over Time](assets/bitchat_relay_count_chart.png) + +### Total Functioning Relays + +The chart below shows all functioning relays discovered during the relay discovery process: + +![Total Functioning Relay Count Over Time](assets/total_relay_count_chart.png) + +The charts are automatically updated daily to reflect changes in the number of relays and show approximately 70 days of history. + ## Attribution `nostr_relays.csv` and `relay_discovery_results.json` use a database curated by DB‑IP, available at https://www.db-ip.com. diff --git a/assets/README.md b/assets/README.md new file mode 100644 index 0000000..726dcb3 --- /dev/null +++ b/assets/README.md @@ -0,0 +1,14 @@ +# Georelays Assets + +This directory contains generated assets for the Georelays project: + +## bitchat_relay_count_chart.png + +A chart showing the history of the number of BitChat-compatible relays (supporting kind 20000 events) in the `nostr_relays.csv` file over time. + +## total_relay_count_chart.png + +A chart showing the history of the total number of functioning relays discovered in the `relay_discovery_results.json` file over time. + +These charts are automatically generated by the GitHub workflow after each update to the relay data. +They track the last 30 changes to the relay lists and plot them on time series graphs. diff --git a/assets/bitchat_relay_count_chart.png b/assets/bitchat_relay_count_chart.png new file mode 100644 index 0000000..19a0d58 Binary files /dev/null and b/assets/bitchat_relay_count_chart.png differ diff --git a/assets/total_relay_count_chart.png b/assets/total_relay_count_chart.png new file mode 100644 index 0000000..eb52856 Binary files /dev/null and b/assets/total_relay_count_chart.png differ diff --git a/requirements.txt b/requirements.txt index 31b5e2f..8f79324 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ websockets>=12.0 +matplotlib>=3.5.0 +pandas>=1.4.0 +numpy>=1.20.0 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..af61e24 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,25 @@ +# Georelays Scripts + +This directory contains utility scripts for the Georelays project: + +## track_relay_counts.py + +This script analyzes the git history of the `nostr_relays.csv` file to track how the number of relay entries changes over time. + +### Features: +- Retrieves the last 30 commits that modified the relay CSV file +- Counts the number of relay entries in each commit +- Creates a time series chart showing the trend of relay counts +- Saves the chart as `assets/relay_count_chart.png` + +### Usage: +``` +python scripts/track_relay_counts.py +``` + +### Dependencies: +- pandas +- matplotlib +- numpy + +This script is automatically run by the GitHub workflow after each update to the relay data. diff --git a/scripts/track_relay_counts.py b/scripts/track_relay_counts.py new file mode 100755 index 0000000..0d0eba5 --- /dev/null +++ b/scripts/track_relay_counts.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Track Relay Count Changes + +This script analyzes the git history of relay data files to track how +the number of relay entries changes over time. It generates charts showing +the trend over the last 70 commits that modified the files (approximately 70 days). + +Two charts are generated: +1. BitChat Relay Count - from nostr_relays.csv (relays supporting kind 20000) +2. Total Relay Count - from relay_discovery_results.json (all functioning relays) +""" + +import os +import subprocess +import re +import json +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from datetime import datetime +import numpy as np + +# Function to get the count of BitChat relays in a specific commit +def get_relay_count(commit_hash): + """ + Get the count of BitChat relay entries in the CSV file at a specific commit. + + Args: + commit_hash: Git commit hash + + Returns: + int: Number of relay entries (excluding header) + """ + try: + # Get the file content at this commit + result = subprocess.run( + ['git', 'show', f'{commit_hash}:nostr_relays.csv'], + capture_output=True, text=True, check=True + ) + + # Count lines excluding header + lines = result.stdout.strip().split('\n') + # Subtract 1 for the header row + return len(lines) - 1 if lines else 0 + except subprocess.CalledProcessError: + # File might not exist in this commit + return 0 + +# Function to get the count of total functioning relays in a specific commit +def get_total_relay_count(commit_hash): + """ + Get the count of total functioning relays from the JSON file at a specific commit. + + Args: + commit_hash: Git commit hash + + Returns: + int: Number of functioning relays + """ + try: + # Get the file content at this commit + result = subprocess.run( + ['git', 'show', f'{commit_hash}:relay_discovery_results.json'], + capture_output=True, text=True, check=True + ) + + # Parse JSON and get the count from functioning_relays array + data = json.loads(result.stdout) + return len(data.get('functioning_relays', [])) + except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError) as e: + print(f"Error getting total relay count from commit {commit_hash}: {e}") + return 0 + +# Function to extract date from commit message +def extract_date_from_commit(commit_hash): + """ + Extract the date from a commit. + + Args: + commit_hash: Git commit hash + + Returns: + datetime: Commit date + """ + try: + result = subprocess.run( + ['git', 'show', '-s', '--format=%ci', commit_hash], + capture_output=True, text=True, check=True + ) + commit_date = result.stdout.strip() + return datetime.strptime(commit_date, '%Y-%m-%d %H:%M:%S %z') + except Exception as e: + print(f"Error extracting date from commit {commit_hash}: {e}") + return None + +def create_plot(data_frame, title, y_label, output_path): + """ + Create and save a plot from the given data. + + Args: + data_frame: DataFrame containing 'date' and 'count' columns + title: Title for the plot + y_label: Label for Y-axis + output_path: Path to save the plot + """ + if data_frame.empty: + print(f"No data found to generate {title} chart") + return + + plt.figure(figsize=(12, 6)) + plt.plot(data_frame['date'], data_frame['count'], marker='o', linestyle='-', linewidth=2) + + # Add title and labels + plt.title(title, fontsize=16) + plt.xlabel('Date', fontsize=12) + plt.ylabel(y_label, fontsize=12) + + # Format the x-axis to show dates nicely + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + plt.gca().xaxis.set_major_locator(mdates.AutoDateLocator()) + + # Add grid and rotate date labels + plt.grid(True, linestyle='--', alpha=0.7) + plt.xticks(rotation=45) + + # Add data points annotation + for i, row in data_frame.iterrows(): + plt.annotate( + f"{row['count']}", + (row['date'], row['count']), + textcoords="offset points", + xytext=(0, 10), + ha='center' + ) + + # Add current count in the corner + if not data_frame.empty: + latest_count = data_frame['count'].iloc[-1] + plt.annotate( + f"Latest Count: {latest_count}", + xy=(0.02, 0.96), + xycoords='axes fraction', + fontsize=12, + bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.8) + ) + + # Adjust layout and save + plt.tight_layout() + + # Save the chart + plt.savefig(output_path, dpi=300) + + # Close the figure to prevent memory leaks + plt.close() + + # Return summary + return len(data_frame), latest_count if not data_frame.empty else 'N/A' + +def main(): + """ + Main function to track relay count changes and generate charts. + """ + # Change to the root directory of the project if necessary + script_dir = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.dirname(script_dir) + os.chdir(root_dir) + + # Ensure the assets directory exists + os.makedirs('assets', exist_ok=True) + + # Get commits that modified the nostr_relays.csv file + bitchat_result = subprocess.run( + ['git', 'log', '--format=%H', '-n', '70', '--', 'nostr_relays.csv'], + capture_output=True, text=True + ) + bitchat_commit_hashes = bitchat_result.stdout.strip().split('\n') + + # Get commits that modified the relay_discovery_results.json file + total_result = subprocess.run( + ['git', 'log', '--format=%H', '-n', '70', '--', 'relay_discovery_results.json'], + capture_output=True, text=True + ) + total_commit_hashes = total_result.stdout.strip().split('\n') + + # Collect BitChat relay data + bitchat_data = [] + for commit_hash in bitchat_commit_hashes: + if not commit_hash: + continue + date = extract_date_from_commit(commit_hash) + count = get_relay_count(commit_hash) + if date and count > 0: # Only add valid entries + bitchat_data.append({ + 'date': date, + 'commit': commit_hash, + 'count': count + }) + + # Collect total relay data + total_data = [] + for commit_hash in total_commit_hashes: + if not commit_hash: + continue + date = extract_date_from_commit(commit_hash) + count = get_total_relay_count(commit_hash) + if date and count > 0: # Only add valid entries + total_data.append({ + 'date': date, + 'commit': commit_hash, + 'count': count + }) + + # Create DataFrames and sort by date + bitchat_df = pd.DataFrame(bitchat_data).sort_values('date') if bitchat_data else pd.DataFrame() + total_df = pd.DataFrame(total_data).sort_values('date') if total_data else pd.DataFrame() + + # Create and save plots + bitchat_stats = create_plot( + bitchat_df, + 'BitChat-Compatible Relay Count Over Time', + 'Number of BitChat Relays', + 'assets/bitchat_relay_count_chart.png' + ) + + total_stats = create_plot( + total_df, + 'Total Functioning Relay Count Over Time', + 'Number of Functioning Relays', + 'assets/total_relay_count_chart.png' + ) + + # Print summary for log + if bitchat_stats: + print(f"Generated BitChat relay chart with {bitchat_stats[0]} data points") + print(f"Latest BitChat relay count: {bitchat_stats[1]}") + + if total_stats: + print(f"Generated total relay chart with {total_stats[0]} data points") + print(f"Latest total relay count: {total_stats[1]}") + +if __name__ == "__main__": + main()