mirror of
https://github.com/permissionlesstech/georelays.git
synced 2026-07-24 22:25:18 +00:00
add automated relay count tracking with dual trend charts (#8)
* Add relay count tracking workflow with dual charts This commit adds: - New GitHub workflow to track relay counts - Two charts: BitChat-compatible relays and total functioning relays - Extended history for both charts (70 days) - Project structure improvements with scripts and assets directories - Updated documentation in README.md and other files - Dependencies in requirements.txt * Remove overlapping data point labels from relay count charts * Add .venv to gitignore * Add relay location maps and automation workflow * Improve relay maps with proper world map backgrounds using cartopy * Add relay location map to README * update readme
This commit is contained in:
lollerfirst
@@ -0,0 +1,25 @@
|
||||
# Georelays Scripts
|
||||
|
||||
This directory contains utility scripts for the Georelays project:
|
||||
|
||||
## track_relay_counts.py
|
||||
|
||||
This script analyzes the git history of the `nostr_relays.csv` file to track how the number of relay entries changes over time.
|
||||
|
||||
### Features:
|
||||
- Retrieves the last 30 commits that modified the relay CSV file
|
||||
- Counts the number of relay entries in each commit
|
||||
- Creates a time series chart showing the trend of relay counts
|
||||
- Saves the chart as `assets/relay_count_chart.png`
|
||||
|
||||
### Usage:
|
||||
```
|
||||
python scripts/track_relay_counts.py
|
||||
```
|
||||
|
||||
### Dependencies:
|
||||
- pandas
|
||||
- matplotlib
|
||||
- numpy
|
||||
|
||||
This script is automatically run by the GitHub workflow after each update to the relay data.
|
||||
Executable
+252
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate World Map of Nostr Relay Locations
|
||||
|
||||
This script reads the nostr_relays.csv file, which contains information about
|
||||
BitChat-compatible Nostr relays, including their geographical coordinates.
|
||||
It generates both an interactive HTML map and a static PNG image showing
|
||||
the distribution of relays around the world.
|
||||
|
||||
The maps are saved in the assets directory.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import folium
|
||||
from folium.plugins import MarkerCluster
|
||||
import numpy as np
|
||||
from matplotlib.colors import LinearSegmentedColormap
|
||||
import time
|
||||
|
||||
def create_interactive_map(df, output_path):
|
||||
"""
|
||||
Create an interactive HTML map showing relay locations with clustering.
|
||||
|
||||
Args:
|
||||
df: DataFrame containing relay data with Latitude and Longitude columns
|
||||
output_path: Path to save the HTML map
|
||||
"""
|
||||
# Create map centered at (0, 0) with zoom level 2
|
||||
world_map = folium.Map(location=[0, 0], zoom_start=2, tiles='CartoDB positron')
|
||||
|
||||
# Add marker cluster
|
||||
marker_cluster = MarkerCluster().add_to(world_map)
|
||||
|
||||
# Add markers for each relay
|
||||
for idx, row in df.iterrows():
|
||||
folium.Marker(
|
||||
location=[row['Latitude'], row['Longitude']],
|
||||
popup=row['Relay URL'],
|
||||
icon=folium.Icon(color='blue', icon='signal', prefix='fa')
|
||||
).add_to(marker_cluster)
|
||||
|
||||
# Save map
|
||||
world_map.save(output_path)
|
||||
|
||||
return len(df)
|
||||
|
||||
def create_static_map(df, output_path):
|
||||
"""
|
||||
Create a static PNG map showing relay locations on a proper world map.
|
||||
|
||||
Args:
|
||||
df: DataFrame containing relay data with Latitude and Longitude columns
|
||||
output_path: Path to save the PNG map
|
||||
"""
|
||||
import cartopy.crs as ccrs
|
||||
import cartopy.feature as cfeature
|
||||
|
||||
plt.figure(figsize=(15, 10))
|
||||
|
||||
# Create a map with a proper projection
|
||||
ax = plt.axes(projection=ccrs.PlateCarree())
|
||||
|
||||
# Add map features
|
||||
ax.add_feature(cfeature.LAND, facecolor='#E5E5E5')
|
||||
ax.add_feature(cfeature.OCEAN, facecolor='#DDEEFF')
|
||||
ax.add_feature(cfeature.COASTLINE, linewidth=0.5, edgecolor='#999999')
|
||||
ax.add_feature(cfeature.BORDERS, linewidth=0.3, edgecolor='#AAAAAA')
|
||||
ax.add_feature(cfeature.LAKES, facecolor='#DDEEFF', edgecolor='#999999', linewidth=0.5)
|
||||
ax.add_feature(cfeature.RIVERS, edgecolor='#99CCFF', linewidth=0.5)
|
||||
|
||||
# Add grid
|
||||
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
|
||||
linewidth=0.5, color='gray', alpha=0.5, linestyle='--')
|
||||
gl.top_labels = False
|
||||
gl.right_labels = False
|
||||
|
||||
# Set map limits
|
||||
ax.set_extent([-180, 180, -90, 90], crs=ccrs.PlateCarree())
|
||||
|
||||
# Plot relay locations
|
||||
plt.scatter(
|
||||
df['Longitude'],
|
||||
df['Latitude'],
|
||||
alpha=0.8,
|
||||
c='blue',
|
||||
s=30,
|
||||
edgecolor='white',
|
||||
linewidth=0.5,
|
||||
transform=ccrs.PlateCarree()
|
||||
)
|
||||
|
||||
# Add title and labels
|
||||
plt.title('Global Distribution of BitChat-Compatible Nostr Relays', fontsize=16)
|
||||
|
||||
# Add timestamp
|
||||
timestamp = time.strftime("%Y-%m-%d", time.localtime())
|
||||
plt.annotate(
|
||||
f'Generated: {timestamp} | Total Relays: {len(df)}',
|
||||
xy=(0.02, 0.02),
|
||||
xycoords='axes fraction',
|
||||
fontsize=10,
|
||||
bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.8)
|
||||
)
|
||||
|
||||
# Adjust layout and save
|
||||
plt.tight_layout()
|
||||
plt.savefig(output_path, dpi=300, bbox_inches='tight')
|
||||
plt.close()
|
||||
|
||||
return len(df)
|
||||
|
||||
def create_heatmap(df, output_path):
|
||||
"""
|
||||
Create a heatmap visualization showing relay density across the world.
|
||||
|
||||
Args:
|
||||
df: DataFrame containing relay data with Latitude and Longitude columns
|
||||
output_path: Path to save the PNG heatmap
|
||||
"""
|
||||
import cartopy.crs as ccrs
|
||||
import cartopy.feature as cfeature
|
||||
from scipy.ndimage import gaussian_filter
|
||||
|
||||
plt.figure(figsize=(15, 10))
|
||||
|
||||
# Create a map with a proper projection
|
||||
ax = plt.axes(projection=ccrs.PlateCarree())
|
||||
|
||||
# Add map features
|
||||
ax.add_feature(cfeature.LAND, facecolor='#E5E5E5')
|
||||
ax.add_feature(cfeature.OCEAN, facecolor='#DDEEFF')
|
||||
ax.add_feature(cfeature.COASTLINE, linewidth=0.5, edgecolor='#999999')
|
||||
ax.add_feature(cfeature.BORDERS, linewidth=0.3, edgecolor='#AAAAAA')
|
||||
|
||||
# Add grid
|
||||
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
|
||||
linewidth=0.5, color='gray', alpha=0.5, linestyle='--')
|
||||
gl.top_labels = False
|
||||
gl.right_labels = False
|
||||
|
||||
# Set map limits
|
||||
ax.set_extent([-180, 180, -90, 90], crs=ccrs.PlateCarree())
|
||||
|
||||
# Create grid for heatmap
|
||||
x = np.linspace(-180, 180, 360)
|
||||
y = np.linspace(-90, 90, 180)
|
||||
|
||||
# Count relays in each grid cell
|
||||
heatmap = np.zeros((len(y)-1, len(x)-1))
|
||||
|
||||
for _, row in df.iterrows():
|
||||
lon_idx = np.searchsorted(x, row['Longitude']) - 1
|
||||
lat_idx = np.searchsorted(y, row['Latitude']) - 1
|
||||
|
||||
if 0 <= lon_idx < len(x)-1 and 0 <= lat_idx < len(y)-1:
|
||||
heatmap[lat_idx, lon_idx] += 1
|
||||
|
||||
# Apply Gaussian smoothing to heatmap
|
||||
heatmap = gaussian_filter(heatmap, sigma=3)
|
||||
|
||||
# Create custom colormap (blue to white)
|
||||
colors = [(0, 0, 0.8, 0), (0, 0, 1, 0.7), (0.5, 0.5, 1, 0.8), (1, 1, 1, 0.9)]
|
||||
cmap = LinearSegmentedColormap.from_list('custom_blue', colors)
|
||||
|
||||
# Plot heatmap
|
||||
plt.pcolormesh(x, y, heatmap, cmap=cmap, alpha=0.7, transform=ccrs.PlateCarree())
|
||||
|
||||
# Add title
|
||||
plt.title('Global Heatmap of BitChat-Compatible Nostr Relays', fontsize=16)
|
||||
|
||||
# Add timestamp and relay count
|
||||
timestamp = time.strftime("%Y-%m-%d", time.localtime())
|
||||
plt.annotate(
|
||||
f'Generated: {timestamp} | Total Relays: {len(df)}',
|
||||
xy=(0.02, 0.02),
|
||||
xycoords='axes fraction',
|
||||
fontsize=10,
|
||||
bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.8)
|
||||
)
|
||||
|
||||
# Adjust layout and save
|
||||
plt.tight_layout()
|
||||
plt.savefig(output_path, dpi=300, bbox_inches='tight')
|
||||
plt.close()
|
||||
|
||||
return len(df)
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to generate maps of relay locations.
|
||||
"""
|
||||
# Change to the root directory of the project
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
root_dir = os.path.dirname(script_dir)
|
||||
os.chdir(root_dir)
|
||||
|
||||
# Ensure the assets directory exists
|
||||
os.makedirs('assets', exist_ok=True)
|
||||
|
||||
try:
|
||||
# Read the relay data
|
||||
df = pd.read_csv('nostr_relays.csv')
|
||||
|
||||
# Filter out rows with missing or invalid coordinates
|
||||
df = df.dropna(subset=['Latitude', 'Longitude'])
|
||||
|
||||
# Filter out invalid coordinates
|
||||
df = df[(df['Latitude'] >= -90) & (df['Latitude'] <= 90) &
|
||||
(df['Longitude'] >= -180) & (df['Longitude'] <= 180)]
|
||||
|
||||
# Generate maps
|
||||
# 1. Interactive HTML map
|
||||
relay_count_interactive = create_interactive_map(
|
||||
df,
|
||||
'assets/relay_locations_interactive.html'
|
||||
)
|
||||
|
||||
# 2. Static PNG map
|
||||
try:
|
||||
relay_count_static = create_static_map(
|
||||
df,
|
||||
'assets/relay_locations_static.png'
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error creating static map: {e}")
|
||||
relay_count_static = 0
|
||||
|
||||
# 3. Heatmap
|
||||
try:
|
||||
from scipy.ndimage import gaussian_filter
|
||||
relay_count_heatmap = create_heatmap(
|
||||
df,
|
||||
'assets/relay_locations_heatmap.png'
|
||||
)
|
||||
except ImportError:
|
||||
print("Could not create heatmap: scipy not installed")
|
||||
relay_count_heatmap = 0
|
||||
|
||||
# Print summary
|
||||
print(f"Generated interactive map with {relay_count_interactive} relays")
|
||||
if relay_count_static > 0:
|
||||
print(f"Generated static map with {relay_count_static} relays")
|
||||
if relay_count_heatmap > 0:
|
||||
print(f"Generated heatmap with {relay_count_heatmap} relays")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error generating relay maps: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+233
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Track Relay Count Changes
|
||||
|
||||
This script analyzes the git history of relay data files to track how
|
||||
the number of relay entries changes over time. It generates charts showing
|
||||
the trend over the last 70 commits that modified the files (approximately 70 days).
|
||||
|
||||
Two charts are generated:
|
||||
1. BitChat Relay Count - from nostr_relays.csv (relays supporting kind 20000)
|
||||
2. Total Relay Count - from relay_discovery_results.json (all functioning relays)
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
import json
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
from datetime import datetime
|
||||
import numpy as np
|
||||
|
||||
# Function to get the count of BitChat relays in a specific commit
|
||||
def get_relay_count(commit_hash):
|
||||
"""
|
||||
Get the count of BitChat relay entries in the CSV file at a specific commit.
|
||||
|
||||
Args:
|
||||
commit_hash: Git commit hash
|
||||
|
||||
Returns:
|
||||
int: Number of relay entries (excluding header)
|
||||
"""
|
||||
try:
|
||||
# Get the file content at this commit
|
||||
result = subprocess.run(
|
||||
['git', 'show', f'{commit_hash}:nostr_relays.csv'],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
|
||||
# Count lines excluding header
|
||||
lines = result.stdout.strip().split('\n')
|
||||
# Subtract 1 for the header row
|
||||
return len(lines) - 1 if lines else 0
|
||||
except subprocess.CalledProcessError:
|
||||
# File might not exist in this commit
|
||||
return 0
|
||||
|
||||
# Function to get the count of total functioning relays in a specific commit
|
||||
def get_total_relay_count(commit_hash):
|
||||
"""
|
||||
Get the count of total functioning relays from the JSON file at a specific commit.
|
||||
|
||||
Args:
|
||||
commit_hash: Git commit hash
|
||||
|
||||
Returns:
|
||||
int: Number of functioning relays
|
||||
"""
|
||||
try:
|
||||
# Get the file content at this commit
|
||||
result = subprocess.run(
|
||||
['git', 'show', f'{commit_hash}:relay_discovery_results.json'],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
|
||||
# Parse JSON and get the count from functioning_relays array
|
||||
data = json.loads(result.stdout)
|
||||
return len(data.get('functioning_relays', []))
|
||||
except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError) as e:
|
||||
print(f"Error getting total relay count from commit {commit_hash}: {e}")
|
||||
return 0
|
||||
|
||||
# Function to extract date from commit message
|
||||
def extract_date_from_commit(commit_hash):
|
||||
"""
|
||||
Extract the date from a commit.
|
||||
|
||||
Args:
|
||||
commit_hash: Git commit hash
|
||||
|
||||
Returns:
|
||||
datetime: Commit date
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'show', '-s', '--format=%ci', commit_hash],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
commit_date = result.stdout.strip()
|
||||
return datetime.strptime(commit_date, '%Y-%m-%d %H:%M:%S %z')
|
||||
except Exception as e:
|
||||
print(f"Error extracting date from commit {commit_hash}: {e}")
|
||||
return None
|
||||
|
||||
def create_plot(data_frame, title, y_label, output_path):
|
||||
"""
|
||||
Create and save a plot from the given data.
|
||||
|
||||
Args:
|
||||
data_frame: DataFrame containing 'date' and 'count' columns
|
||||
title: Title for the plot
|
||||
y_label: Label for Y-axis
|
||||
output_path: Path to save the plot
|
||||
"""
|
||||
if data_frame.empty:
|
||||
print(f"No data found to generate {title} chart")
|
||||
return
|
||||
|
||||
plt.figure(figsize=(12, 6))
|
||||
plt.plot(data_frame['date'], data_frame['count'], marker='o', linestyle='-', linewidth=2)
|
||||
|
||||
# Add title and labels
|
||||
plt.title(title, fontsize=16)
|
||||
plt.xlabel('Date', fontsize=12)
|
||||
plt.ylabel(y_label, fontsize=12)
|
||||
|
||||
# Format the x-axis to show dates nicely
|
||||
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
|
||||
plt.gca().xaxis.set_major_locator(mdates.AutoDateLocator())
|
||||
|
||||
# Add grid and rotate date labels
|
||||
plt.grid(True, linestyle='--', alpha=0.7)
|
||||
plt.xticks(rotation=45)
|
||||
|
||||
# Add current count in the corner
|
||||
if not data_frame.empty:
|
||||
latest_count = data_frame['count'].iloc[-1]
|
||||
plt.annotate(
|
||||
f"Latest Count: {latest_count}",
|
||||
xy=(0.02, 0.96),
|
||||
xycoords='axes fraction',
|
||||
fontsize=12,
|
||||
bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.8)
|
||||
)
|
||||
|
||||
# Adjust layout and save
|
||||
plt.tight_layout()
|
||||
|
||||
# Save the chart
|
||||
plt.savefig(output_path, dpi=300)
|
||||
|
||||
# Close the figure to prevent memory leaks
|
||||
plt.close()
|
||||
|
||||
# Return summary
|
||||
return len(data_frame), latest_count if not data_frame.empty else 'N/A'
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to track relay count changes and generate charts.
|
||||
"""
|
||||
# Change to the root directory of the project if necessary
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
root_dir = os.path.dirname(script_dir)
|
||||
os.chdir(root_dir)
|
||||
|
||||
# Ensure the assets directory exists
|
||||
os.makedirs('assets', exist_ok=True)
|
||||
|
||||
# Get commits that modified the nostr_relays.csv file
|
||||
bitchat_result = subprocess.run(
|
||||
['git', 'log', '--format=%H', '-n', '70', '--', 'nostr_relays.csv'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
bitchat_commit_hashes = bitchat_result.stdout.strip().split('\n')
|
||||
|
||||
# Get commits that modified the relay_discovery_results.json file
|
||||
total_result = subprocess.run(
|
||||
['git', 'log', '--format=%H', '-n', '70', '--', 'relay_discovery_results.json'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
total_commit_hashes = total_result.stdout.strip().split('\n')
|
||||
|
||||
# Collect BitChat relay data
|
||||
bitchat_data = []
|
||||
for commit_hash in bitchat_commit_hashes:
|
||||
if not commit_hash:
|
||||
continue
|
||||
date = extract_date_from_commit(commit_hash)
|
||||
count = get_relay_count(commit_hash)
|
||||
if date and count > 0: # Only add valid entries
|
||||
bitchat_data.append({
|
||||
'date': date,
|
||||
'commit': commit_hash,
|
||||
'count': count
|
||||
})
|
||||
|
||||
# Collect total relay data
|
||||
total_data = []
|
||||
for commit_hash in total_commit_hashes:
|
||||
if not commit_hash:
|
||||
continue
|
||||
date = extract_date_from_commit(commit_hash)
|
||||
count = get_total_relay_count(commit_hash)
|
||||
if date and count > 0: # Only add valid entries
|
||||
total_data.append({
|
||||
'date': date,
|
||||
'commit': commit_hash,
|
||||
'count': count
|
||||
})
|
||||
|
||||
# Create DataFrames and sort by date
|
||||
bitchat_df = pd.DataFrame(bitchat_data).sort_values('date') if bitchat_data else pd.DataFrame()
|
||||
total_df = pd.DataFrame(total_data).sort_values('date') if total_data else pd.DataFrame()
|
||||
|
||||
# Create and save plots
|
||||
bitchat_stats = create_plot(
|
||||
bitchat_df,
|
||||
'BitChat-Compatible Relay Count Over Time',
|
||||
'Number of BitChat Relays',
|
||||
'assets/bitchat_relay_count_chart.png'
|
||||
)
|
||||
|
||||
total_stats = create_plot(
|
||||
total_df,
|
||||
'Total Functioning Relay Count Over Time',
|
||||
'Number of Functioning Relays',
|
||||
'assets/total_relay_count_chart.png'
|
||||
)
|
||||
|
||||
# Print summary for log
|
||||
if bitchat_stats:
|
||||
print(f"Generated BitChat relay chart with {bitchat_stats[0]} data points")
|
||||
print(f"Latest BitChat relay count: {bitchat_stats[1]}")
|
||||
|
||||
if total_stats:
|
||||
print(f"Generated total relay chart with {total_stats[0]} data points")
|
||||
print(f"Latest total relay count: {total_stats[1]}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user