ccsite-manager: fleet orchestrator, BTCPay auto-settlement, watchdog, dashboard
This commit is contained in:
drjones
216
ccsite.py
Normal file
216
ccsite.py
Normal file
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ccsite-manager — Mass orchestration CLI for the 10-shop fleet.
|
||||
Usage:
|
||||
ccsite push <local_path> <remote_path> [--site N] Push a file to all/specific sites
|
||||
ccsite cmd "<command>" [--site N] Run a shell command on all sites
|
||||
ccsite restart [--site N] Restart the shop service
|
||||
ccsite sql "<SQL>" [--site N] Run SQLite query on all sites
|
||||
ccsite list List all sites with status
|
||||
ccsite health Health check all sites
|
||||
ccsite push-template <name> <content> [--site N] Push a Jinja2 template to all sites
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
SITES = {
|
||||
1: {"ip": "10.30.20.210", "vmid": "123", "name": "Digital Marketplace"},
|
||||
2: {"ip": "10.30.20.211", "vmid": "124", "name": "CardVault Pro"},
|
||||
3: {"ip": "10.30.20.212", "vmid": "125", "name": "StreamPass Hub"},
|
||||
4: {"ip": "10.30.20.213", "vmid": "126", "name": "KeyForge Digital"},
|
||||
5: {"ip": "10.30.20.214", "vmid": "127", "name": "CoinBridge Market"},
|
||||
6: {"ip": "10.30.20.217", "vmid": "128", "name": "GiftShift"},
|
||||
7: {"ip": "10.30.20.218", "vmid": "129", "name": "PremiumPortal"},
|
||||
8: {"ip": "10.30.20.219", "vmid": "130", "name": "DigiDeals"},
|
||||
9: {"ip": "10.30.20.220", "vmid": "131", "name": "NexusKeys"},
|
||||
10: {"ip": "10.30.20.221", "vmid": "132", "name": "VaultPass"},
|
||||
}
|
||||
|
||||
PROXMOX = "root@10.30.20.85"
|
||||
SHOP_DIR = "/opt/shop"
|
||||
SHOP_DB = "/opt/shop/shop.db"
|
||||
|
||||
def ssh_cmd(ip, cmd, timeout=15):
|
||||
"""Run a command on a site via direct SSH."""
|
||||
full = f"ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 root@{ip} '{cmd}'"
|
||||
try:
|
||||
r = subprocess.run(full, shell=True, capture_output=True, text=True, timeout=timeout)
|
||||
return {"ok": r.returncode == 0, "stdout": r.stdout.strip(), "stderr": r.stderr.strip()}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"ok": False, "stdout": "", "stderr": "TIMEOUT"}
|
||||
except Exception as e:
|
||||
return {"ok": False, "stdout": "", "stderr": str(e)}
|
||||
|
||||
def pct_exec(vmid, cmd, timeout=15):
|
||||
"""Run a command inside a container via Proxmox pct exec."""
|
||||
safe_cmd = cmd.replace("'", "'\"'\"'")
|
||||
full = f"ssh {PROXMOX} \"pct exec {vmid} -- bash -c '{safe_cmd}'\""
|
||||
try:
|
||||
r = subprocess.run(full, shell=True, capture_output=True, text=True, timeout=timeout)
|
||||
return {"ok": r.returncode == 0, "stdout": r.stdout.strip(), "stderr": r.stderr.strip()}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"ok": False, "stdout": "", "stderr": "TIMEOUT"}
|
||||
except Exception as e:
|
||||
return {"ok": False, "stdout": "", "stderr": str(e)}
|
||||
|
||||
def push_file(ip, local_path, remote_path, timeout=15):
|
||||
"""Push a file to a site via base64 + SSH."""
|
||||
with open(local_path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode()
|
||||
cmd = f"echo '{b64}' | base64 -d > {remote_path}"
|
||||
return ssh_cmd(ip, cmd, timeout)
|
||||
|
||||
def push_content(ip, content, remote_path, timeout=15):
|
||||
"""Push raw content to a site."""
|
||||
b64 = base64.b64encode(content.encode()).decode()
|
||||
cmd = f"echo '{b64}' | base64 -d > {remote_path}"
|
||||
return ssh_cmd(ip, cmd, timeout)
|
||||
|
||||
def run_on_sites(sites_filter=None, func=None, desc=""):
|
||||
"""Run a function across sites in parallel. Returns dict of results."""
|
||||
targets = {k: v for k, v in SITES.items() if sites_filter is None or k in sites_filter}
|
||||
results = {}
|
||||
with ThreadPoolExecutor(max_workers=10) as ex:
|
||||
futures = {ex.submit(func, s): (i, s) for i, s in targets.items()}
|
||||
for f in as_completed(futures):
|
||||
i, s = futures[f]
|
||||
try:
|
||||
results[i] = f.result()
|
||||
except Exception as e:
|
||||
results[i] = {"ok": False, "stderr": str(e)}
|
||||
status = "✓" if results[i].get("ok") else "✗"
|
||||
print(f" [{status}] ccsite{i} ({s['name']}) — {desc}")
|
||||
return results
|
||||
|
||||
# ─── Commands ─────────────────────────────────────────────
|
||||
|
||||
def cmd_list():
|
||||
print(f"{'#':<3} {'Name':<25} {'IP':<16} {'VMID':<6} Status")
|
||||
print("-" * 60)
|
||||
def check(s):
|
||||
r = ssh_cmd(s['ip'], "systemctl is-active shop 2>/dev/null || echo dead")
|
||||
status = r.get('stdout', '?').strip()
|
||||
return r
|
||||
results = run_on_sites(func=check, desc="status check")
|
||||
for i in sorted(SITES):
|
||||
s = SITES[i]
|
||||
r = results.get(i, {})
|
||||
status = r.get('stdout', '?').strip()
|
||||
status_icon = "🟢" if status == "active" else "🔴"
|
||||
print(f"{i:<3} {s['name']:<25} {s['ip']:<16} {s['vmid']:<6} {status_icon} {status}")
|
||||
|
||||
def cmd_health():
|
||||
print("Health check across all sites...")
|
||||
def check(s):
|
||||
try:
|
||||
r = subprocess.run(
|
||||
f"curl -s --connect-timeout 3 http://{s['ip']}:5000/health",
|
||||
shell=True, capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if r.returncode == 0:
|
||||
d = json.loads(r.stdout)
|
||||
return {"ok": True, "stdout": f"{d.get('products', '?')} products, {d.get('users', '?')} users"}
|
||||
return {"ok": False, "stdout": r.stderr or "no response"}
|
||||
except:
|
||||
return {"ok": False, "stdout": "connection failed"}
|
||||
results = run_on_sites(func=check, desc="health")
|
||||
ok = sum(1 for r in results.values() if r.get("ok"))
|
||||
print(f"\n{ok}/10 sites healthy")
|
||||
|
||||
def cmd_restart(sites_filter=None):
|
||||
print("Restarting shop service...")
|
||||
def restart(s):
|
||||
return ssh_cmd(s['ip'], "systemctl restart shop && sleep 1 && systemctl is-active shop")
|
||||
run_on_sites(sites_filter, restart, "restart")
|
||||
|
||||
def cmd_push(args):
|
||||
local = args[0]
|
||||
remote = args[1] if len(args) > 1 else f"{SHOP_DIR}/{Path(local).name}"
|
||||
sites_filter = parse_site_filter(args)
|
||||
print(f"Pushing {local} → {remote}")
|
||||
def push(s):
|
||||
return push_file(s['ip'], local, remote)
|
||||
results = run_on_sites(sites_filter, push, f"→ {remote}")
|
||||
ok = sum(1 for r in results.values() if r.get("ok"))
|
||||
print(f"\nPushed to {ok}/{len(results)} sites")
|
||||
# Restart if pushing to templates or app
|
||||
if 'templates' in remote or 'app.py' in remote:
|
||||
print("Code change detected — restarting services...")
|
||||
cmd_restart(sites_filter)
|
||||
|
||||
def cmd_cmd(args):
|
||||
if not args:
|
||||
print("Usage: ccsite cmd '<command>' [--site N]")
|
||||
return
|
||||
cmd_str = args[0]
|
||||
sites_filter = parse_site_filter(args)
|
||||
print(f"Running: {cmd_str}")
|
||||
def run(s):
|
||||
return ssh_cmd(s['ip'], cmd_str, timeout=20)
|
||||
results = run_on_sites(sites_filter, run, cmd_str[:40])
|
||||
for i in sorted(results):
|
||||
r = results[i]
|
||||
out = r.get('stdout', '') or r.get('stderr', '')
|
||||
if out:
|
||||
print(f"\n── ccsite{i} ({SITES[i]['name']}) ──")
|
||||
print(out[:500])
|
||||
|
||||
def cmd_sql(args):
|
||||
if not args:
|
||||
print("Usage: ccsite sql '<SQL query>' [--site N]")
|
||||
return
|
||||
sql = args[0]
|
||||
sites_filter = parse_site_filter(args)
|
||||
print(f"SQL: {sql}")
|
||||
def run(s):
|
||||
return ssh_cmd(s['ip'], f"sqlite3 {SHOP_DB} \"{sql}\" 2>&1", timeout=10)
|
||||
results = run_on_sites(sites_filter, run, sql[:40])
|
||||
for i in sorted(results):
|
||||
r = results[i]
|
||||
out = r.get('stdout', '') or r.get('stderr', '')
|
||||
print(f"\n── ccsite{i} ({SITES[i]['name']}) ──")
|
||||
print(out[:500] if out else "(no results)")
|
||||
|
||||
def parse_site_filter(args):
|
||||
"""Extract --site N from args. Returns set of site IDs or None for all."""
|
||||
for a in args:
|
||||
if a.startswith('--site='):
|
||||
try:
|
||||
return {int(a.split('=')[1])}
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def print_usage():
|
||||
print(__doc__)
|
||||
|
||||
# ─── Main ──────────────────────────────────────────────────
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print_usage()
|
||||
sys.exit(1)
|
||||
|
||||
command = sys.argv[1]
|
||||
rest = sys.argv[2:]
|
||||
|
||||
if command == 'list':
|
||||
cmd_list()
|
||||
elif command == 'health':
|
||||
cmd_health()
|
||||
elif command == 'restart':
|
||||
cmd_restart(parse_site_filter(rest))
|
||||
elif command == 'push':
|
||||
cmd_push(rest)
|
||||
elif command == 'cmd':
|
||||
cmd_cmd(rest)
|
||||
elif command == 'sql':
|
||||
cmd_sql(rest)
|
||||
else:
|
||||
print(f"Unknown command: {command}")
|
||||
print_usage()
|
||||
Reference in New Issue
Block a user