drjones/ccsite-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ccsite-manager: fleet orchestrator, BTCPay auto-settlement, watchdog, dashboard

Browse Source
This commit is contained in:
drjones
2026-08-02 22:36:59 -07:00
commit 4477d56b30
15 changed files with 2754 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
watchdog.py Normal file
View File

@@ -0,0 +1,215 @@
#!/usr/bin/env python3
"""
CCSite Watchdog — Self-healing health monitor.
Extends the dashboard with automated checks, self-healing, and alerts.
"""
import json, time, sqlite3, subprocess, threading, sys
from datetime import datetime
import requests, urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# ─── Config ───────────────────────────────────────────────
SITES = {
1: {"name": "VaultPass", "ip": "10.30.20.210", "emoji": "🎮", "vmid": 123},
2: {"name": "StreamPass", "ip": "10.30.20.211", "emoji": "🎬", "vmid": 124},
3: {"name": "KeyForge", "ip": "10.30.20.212", "emoji": "💻", "vmid": 125},
4: {"name": "SocialForge", "ip": "10.30.20.213", "emoji": "🐦", "vmid": 126},
5: {"name": "CardVault", "ip": "10.30.20.214", "emoji": "💳", "vmid": 127},
6: {"name": "CloudNexus", "ip": "10.30.20.217", "emoji": "☁️", "vmid": 128},
7: {"name": "LearnForge", "ip": "10.30.20.218", "emoji": "📚", "vmid": 129},
8: {"name": "PrivacyPass", "ip": "10.30.20.219", "emoji": "🛡️", "vmid": 130},
9: {"name": "CreatorForge", "ip": "10.30.20.220", "emoji": "🎨", "vmid": 131},
10: {"name": "FoodVault", "ip": "10.30.20.221", "emoji": "🍕", "vmid": 132},
}
BTCPAY_URL = "https://10.30.20.140"
BTCPAY_KEY = "6026288e2e315984661c748baafd509e81a75f22"
BTCPAY_STORES = {
1: "AYZWEAt63TgY4QaxwNBawHASMKcimJabcZxSczRqZG81",
2: "5pi94zmY3F26pTZJ99JfsFG9pb8atbz7fpALrPjS2BRq",
3: "3mc58myPrkdN7hRFdsBwJxbVgBQQ3VvSYZygKQC3FiSG",
4: "E1uZ5nmU6qSpMB8sna7fNdxZFiem9JmkxB9LdJkFN23B",
5: "BcvaUvv8MM8if5dzqe5yLnNERG6EwjjmeMok54ATqjWf",
6: "GvSsA9xaqymhswsiPo4APHMuT2sXCCtnBLgPWcHigxVx",
7: "A2TXbpkpj88Wtj34dLS9yzjMgmcodysEaRhTjc3rFcUr",
8: "9N8dTC99uXn3W3e2F9o8kVQ5ewijh87NevKmMATF1Jcd",
9: "39BKRF3zQb6Gi8LiKo1US13bzJfhpCzkwoHP6pCyQFwD",
10: "9vCjKrKbwgey4GH7DBAMa6FgkiApgHvd5khFpWSZzJL2",
}
SSH = "sshpass -p 'Czapiewski1!' ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@10.30.20.85"
watchdog_state = {"checks": [], "issues": [], "healing_actions": [], "last_check": None}
state_lock = threading.Lock()
# ─── Checks ────────────────────────────────────────────────
def check_site_http(site_id, info):
"""Check if a site returns HTTP 200."""
try:
r = requests.get(f"http://{info['ip']}:5000/", timeout=5)
return {"check": f"{info['emoji']} {info['name']} HTTP", "status": "pass" if r.status_code == 200 else "warn", "detail": f"HTTP {r.status_code}"}
except:
return {"check": f"{info['emoji']} {info['name']} HTTP", "status": "fail", "detail": "unreachable"}
def check_site_btcpay_config(site_id, info):
"""Check if the site is reachable (BTCPay config verified separately)."""
try:
r = requests.get(f"http://{info['ip']}:5000/", timeout=5)
return {"check": f"{info['emoji']} {info['name']} Site", "status": "pass" if r.status_code == 200 else "warn", "detail": f"HTTP {r.status_code}"}
except:
return {"check": f"{info['emoji']} {info['name']} BTCPay config", "status": "fail", "detail": "unreachable"}
def check_btcpay_sync():
"""Check BTCPay sync status."""
try:
r = requests.get(f"{BTCPAY_URL}/api/v1/server/info", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=10, verify=False)
if r.status_code == 200:
info = r.json()
synced = info.get("fullySynched", False)
for s in info.get("syncStatus", []):
ni = s.get("nodeInformation", {})
if ni.get("headers"):
pct = (ni["blocks"] / ni["headers"] * 100) if ni["headers"] else 0
return {"check": "₿ BTCPay Sync", "status": "pass" if synced else "warn", "detail": f"{pct:.1f}% ({ni['blocks']}/{ni['headers']})"}
return {"check": "₿ BTCPay Sync", "status": "pass" if synced else "warn", "detail": f"synced={synced}"}
return {"check": "₿ BTCPay Sync", "status": "fail", "detail": f"HTTP {r.status_code}"}
except Exception as e:
return {"check": "₿ BTCPay Sync", "status": "fail", "detail": str(e)[:80]}
def check_wallet_health():
"""Check how many stores have working wallets by creating test invoices."""
healthy = 0
for sid, store_id in BTCPAY_STORES.items():
try:
# Try creating a 0.01 invoice — if wallet works, it'll succeed
r = requests.post(f"{BTCPAY_URL}/api/v1/stores/{store_id}/invoices",
headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"},
json={"amount": "5.00", "currency": "USD"}, timeout=8, verify=False)
if r.status_code == 200:
healthy += 1
except:
pass
total = len(BTCPAY_STORES)
return {"check": "₿ Wallet Health", "status": "pass" if healthy == total else "warn" if healthy > 0 else "fail",
"detail": f"{healthy}/{total} stores can create invoices"}
def check_webhook_health():
"""Check webhooks are enabled on all stores."""
ok = 0
total = len(BTCPAY_STORES)
for sid, store_id in BTCPAY_STORES.items():
try:
r = requests.get(f"{BTCPAY_URL}/api/v1/stores/{store_id}/webhooks", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=5, verify=False)
if r.status_code == 200:
hooks = r.json()
if hooks and hooks[0].get("enabled"):
ok += 1
except:
pass
return {"check": "🔗 Webhooks", "status": "pass" if ok == total else "warn" if ok > 0 else "fail", "detail": f"{ok}/{total} enabled"}
# ─── Self-Healing ──────────────────────────────────────────
def heal_restart_site(site_id, info):
"""Restart a site's shop service."""
try:
result = subprocess.run(f"{SSH} 'pct exec {info['vmid']} -- systemctl restart shop'",
shell=True, capture_output=True, text=True, timeout=15)
success = result.returncode == 0
return {"action": f"Restart {info['name']}", "success": success, "detail": "restarted" if success else result.stderr[:80]}
except Exception as e:
return {"action": f"Restart {info['name']}", "success": False, "detail": str(e)[:80]}
def heal_generate_wallet(store_id, site_name):
"""Try to generate a wallet for a store."""
try:
r = requests.post(f"{BTCPAY_URL}/api/v1/stores/{store_id}/payment-methods/BTC-CHAIN/wallet/generate",
headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"},
json={"savePrivateKeys": True}, timeout=15, verify=False)
if r.status_code == 200:
return {"action": f"Generate wallet for {site_name}", "success": True, "detail": "generated"}
return {"action": f"Generate wallet for {site_name}", "success": False, "detail": f"HTTP {r.status_code}: {r.text[:80]}"}
except Exception as e:
return {"action": f"Generate wallet for {site_name}", "success": False, "detail": str(e)[:80]}
# ─── Main Check Loop ──────────────────────────────────────
def run_all_checks():
"""Run all watchdog checks and trigger healing if needed."""
checks = []
issues = []
healing = []
# Site HTTP checks (one per site)
for sid, info in SITES.items():
http_check = check_site_http(sid, info)
checks.append(http_check)
if http_check["status"] == "fail":
issues.append(f"{info['emoji']} {info['name']} is DOWN")
result = heal_restart_site(sid, info)
healing.append(result)
# BTCPay health
sync_check = check_btcpay_sync()
checks.append(sync_check)
if sync_check["status"] == "fail":
issues.append("BTCPay is DOWN")
wallet_check = check_wallet_health()
checks.append(wallet_check)
if wallet_check["status"] == "warn":
# Try healing missing wallets
for sid, store_id in BTCPAY_STORES.items():
try:
r = requests.get(f"{BTCPAY_URL}/api/v1/stores/{store_id}/payment-methods/BTC-CHAIN", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=5, verify=False)
if r.status_code == 200:
pm = r.json()
if not pm.get("walletId") and pm.get("derivationScheme") in (None, "", "NONE"):
result = heal_generate_wallet(store_id, SITES[sid]["name"])
healing.append(result)
except:
pass
webhook_check = check_webhook_health()
checks.append(webhook_check)
# Summary
pass_count = sum(1 for c in checks if c["status"] == "pass")
warn_count = sum(1 for c in checks if c["status"] == "warn")
fail_count = sum(1 for c in checks if c["status"] == "fail")
with state_lock:
watchdog_state["checks"] = checks
watchdog_state["issues"] = issues
watchdog_state["healing_actions"] = healing
watchdog_state["last_check"] = datetime.now().strftime("%H:%M:%S")
watchdog_state["summary"] = {
"total": len(checks), "pass": pass_count, "warn": warn_count, "fail": fail_count,
"status": "healthy" if fail_count == 0 and warn_count == 0 else "degraded" if fail_count == 0 else "critical"
}
return watchdog_state
def watchdog_loop():
"""Background thread — runs checks every 60 seconds."""
while True:
try:
run_all_checks()
except Exception as e:
print(f"Watchdog error: {e}", file=sys.stderr)
time.sleep(60)
# ─── Flask Integration ────────────────────────────────────
def get_state():
with state_lock:
return dict(watchdog_state)
if __name__ == '__main__':
print("Watchdog starting...")
run_all_checks()
print(f" Checks: {watchdog_state['summary']}")
t = threading.Thread(target=watchdog_loop, daemon=True)
t.start()
# Keep alive
while True:
time.sleep(60)