#!/usr/bin/env python3 """ CCSite Watchdog — Self-healing health monitor. Extends the dashboard with automated checks, self-healing, and alerts. """ import json, time, sqlite3, subprocess, threading, sys from datetime import datetime import requests, urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # ─── Config ─────────────────────────────────────────────── SITES = { 1: {"name": "VaultPass", "ip": "10.30.20.210", "emoji": "🎮", "vmid": 123}, 2: {"name": "StreamPass", "ip": "10.30.20.211", "emoji": "🎬", "vmid": 124}, 3: {"name": "KeyForge", "ip": "10.30.20.212", "emoji": "💻", "vmid": 125}, 4: {"name": "SocialForge", "ip": "10.30.20.213", "emoji": "🐦", "vmid": 126}, 5: {"name": "CardVault", "ip": "10.30.20.214", "emoji": "💳", "vmid": 127}, 6: {"name": "CloudNexus", "ip": "10.30.20.217", "emoji": "☁️", "vmid": 128}, 7: {"name": "LearnForge", "ip": "10.30.20.218", "emoji": "📚", "vmid": 129}, 8: {"name": "PrivacyPass", "ip": "10.30.20.219", "emoji": "🛡️", "vmid": 130}, 9: {"name": "CreatorForge", "ip": "10.30.20.220", "emoji": "🎨", "vmid": 131}, 10: {"name": "FoodVault", "ip": "10.30.20.221", "emoji": "🍕", "vmid": 132}, } BTCPAY_URL = "https://10.30.20.140" BTCPAY_KEY = "6026288e2e315984661c748baafd509e81a75f22" BTCPAY_STORES = { 1: "AYZWEAt63TgY4QaxwNBawHASMKcimJabcZxSczRqZG81", 2: "5pi94zmY3F26pTZJ99JfsFG9pb8atbz7fpALrPjS2BRq", 3: "3mc58myPrkdN7hRFdsBwJxbVgBQQ3VvSYZygKQC3FiSG", 4: "E1uZ5nmU6qSpMB8sna7fNdxZFiem9JmkxB9LdJkFN23B", 5: "BcvaUvv8MM8if5dzqe5yLnNERG6EwjjmeMok54ATqjWf", 6: "GvSsA9xaqymhswsiPo4APHMuT2sXCCtnBLgPWcHigxVx", 7: "A2TXbpkpj88Wtj34dLS9yzjMgmcodysEaRhTjc3rFcUr", 8: "9N8dTC99uXn3W3e2F9o8kVQ5ewijh87NevKmMATF1Jcd", 9: "39BKRF3zQb6Gi8LiKo1US13bzJfhpCzkwoHP6pCyQFwD", 10: "9vCjKrKbwgey4GH7DBAMa6FgkiApgHvd5khFpWSZzJL2", } SSH = "sshpass -p 'Czapiewski1!' ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@10.30.20.85" watchdog_state = {"checks": [], "issues": [], "healing_actions": [], "last_check": None} state_lock = threading.Lock() # ─── Checks ──────────────────────────────────────────────── def check_site_http(site_id, info): """Check if a site returns HTTP 200.""" try: r = requests.get(f"http://{info['ip']}:5000/", timeout=5) return {"check": f"{info['emoji']} {info['name']} HTTP", "status": "pass" if r.status_code == 200 else "warn", "detail": f"HTTP {r.status_code}"} except: return {"check": f"{info['emoji']} {info['name']} HTTP", "status": "fail", "detail": "unreachable"} def check_site_btcpay_config(site_id, info): """Check if the site is reachable (BTCPay config verified separately).""" try: r = requests.get(f"http://{info['ip']}:5000/", timeout=5) return {"check": f"{info['emoji']} {info['name']} Site", "status": "pass" if r.status_code == 200 else "warn", "detail": f"HTTP {r.status_code}"} except: return {"check": f"{info['emoji']} {info['name']} BTCPay config", "status": "fail", "detail": "unreachable"} def check_btcpay_sync(): """Check BTCPay sync status.""" try: r = requests.get(f"{BTCPAY_URL}/api/v1/server/info", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=10, verify=False) if r.status_code == 200: info = r.json() synced = info.get("fullySynched", False) for s in info.get("syncStatus", []): ni = s.get("nodeInformation", {}) if ni.get("headers"): pct = (ni["blocks"] / ni["headers"] * 100) if ni["headers"] else 0 return {"check": "₿ BTCPay Sync", "status": "pass" if synced else "warn", "detail": f"{pct:.1f}% ({ni['blocks']}/{ni['headers']})"} return {"check": "₿ BTCPay Sync", "status": "pass" if synced else "warn", "detail": f"synced={synced}"} return {"check": "₿ BTCPay Sync", "status": "fail", "detail": f"HTTP {r.status_code}"} except Exception as e: return {"check": "₿ BTCPay Sync", "status": "fail", "detail": str(e)[:80]} def check_wallet_health(): """Check how many stores have working wallets by creating test invoices.""" healthy = 0 for sid, store_id in BTCPAY_STORES.items(): try: # Try creating a 0.01 invoice — if wallet works, it'll succeed r = requests.post(f"{BTCPAY_URL}/api/v1/stores/{store_id}/invoices", headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"}, json={"amount": "5.00", "currency": "USD"}, timeout=8, verify=False) if r.status_code == 200: healthy += 1 except: pass total = len(BTCPAY_STORES) return {"check": "₿ Wallet Health", "status": "pass" if healthy == total else "warn" if healthy > 0 else "fail", "detail": f"{healthy}/{total} stores can create invoices"} def check_webhook_health(): """Check webhooks are enabled on all stores.""" ok = 0 total = len(BTCPAY_STORES) for sid, store_id in BTCPAY_STORES.items(): try: r = requests.get(f"{BTCPAY_URL}/api/v1/stores/{store_id}/webhooks", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=5, verify=False) if r.status_code == 200: hooks = r.json() if hooks and hooks[0].get("enabled"): ok += 1 except: pass return {"check": "🔗 Webhooks", "status": "pass" if ok == total else "warn" if ok > 0 else "fail", "detail": f"{ok}/{total} enabled"} # ─── Self-Healing ────────────────────────────────────────── def heal_restart_site(site_id, info): """Restart a site's shop service.""" try: result = subprocess.run(f"{SSH} 'pct exec {info['vmid']} -- systemctl restart shop'", shell=True, capture_output=True, text=True, timeout=15) success = result.returncode == 0 return {"action": f"Restart {info['name']}", "success": success, "detail": "restarted" if success else result.stderr[:80]} except Exception as e: return {"action": f"Restart {info['name']}", "success": False, "detail": str(e)[:80]} def heal_generate_wallet(store_id, site_name): """Try to generate a wallet for a store.""" try: r = requests.post(f"{BTCPAY_URL}/api/v1/stores/{store_id}/payment-methods/BTC-CHAIN/wallet/generate", headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"}, json={"savePrivateKeys": True}, timeout=15, verify=False) if r.status_code == 200: return {"action": f"Generate wallet for {site_name}", "success": True, "detail": "generated"} return {"action": f"Generate wallet for {site_name}", "success": False, "detail": f"HTTP {r.status_code}: {r.text[:80]}"} except Exception as e: return {"action": f"Generate wallet for {site_name}", "success": False, "detail": str(e)[:80]} # ─── Main Check Loop ────────────────────────────────────── def run_all_checks(): """Run all watchdog checks and trigger healing if needed.""" checks = [] issues = [] healing = [] # Site HTTP checks (one per site) for sid, info in SITES.items(): http_check = check_site_http(sid, info) checks.append(http_check) if http_check["status"] == "fail": issues.append(f"{info['emoji']} {info['name']} is DOWN") result = heal_restart_site(sid, info) healing.append(result) # BTCPay health sync_check = check_btcpay_sync() checks.append(sync_check) if sync_check["status"] == "fail": issues.append("BTCPay is DOWN") wallet_check = check_wallet_health() checks.append(wallet_check) if wallet_check["status"] == "warn": # Try healing missing wallets for sid, store_id in BTCPAY_STORES.items(): try: r = requests.get(f"{BTCPAY_URL}/api/v1/stores/{store_id}/payment-methods/BTC-CHAIN", headers={"Authorization": f"token {BTCPAY_KEY}"}, timeout=5, verify=False) if r.status_code == 200: pm = r.json() if not pm.get("walletId") and pm.get("derivationScheme") in (None, "", "NONE"): result = heal_generate_wallet(store_id, SITES[sid]["name"]) healing.append(result) except: pass webhook_check = check_webhook_health() checks.append(webhook_check) # Summary pass_count = sum(1 for c in checks if c["status"] == "pass") warn_count = sum(1 for c in checks if c["status"] == "warn") fail_count = sum(1 for c in checks if c["status"] == "fail") with state_lock: watchdog_state["checks"] = checks watchdog_state["issues"] = issues watchdog_state["healing_actions"] = healing watchdog_state["last_check"] = datetime.now().strftime("%H:%M:%S") watchdog_state["summary"] = { "total": len(checks), "pass": pass_count, "warn": warn_count, "fail": fail_count, "status": "healthy" if fail_count == 0 and warn_count == 0 else "degraded" if fail_count == 0 else "critical" } return watchdog_state def watchdog_loop(): """Background thread — runs checks every 60 seconds.""" while True: try: run_all_checks() except Exception as e: print(f"Watchdog error: {e}", file=sys.stderr) time.sleep(60) # ─── Flask Integration ──────────────────────────────────── def get_state(): with state_lock: return dict(watchdog_state) if __name__ == '__main__': print("Watchdog starting...") run_all_checks() print(f" Checks: {watchdog_state['summary']}") t = threading.Thread(target=watchdog_loop, daemon=True) t.start() # Keep alive while True: time.sleep(60)