#!/usr/bin/env python3 """K4LSH1_OPS Watchdog — runs the same checks the cron job does every 4 hours. Can be run manually: .venv/bin/python watchdog_test.py""" import json, sqlite3, subprocess, sys, time from pathlib import Path HOME = Path(__file__).parent COINS = { "doge": "DOGE", "sol": "SOL", "eth": "ETH", "xrp": "XRP", "btc": "BTC", "bnb": "BNB", "near": "NEAR", "zec": "ZEC", } PASS = FAIL = 0 issues = [] def ok(msg, condition, detail=""): global PASS, FAIL if condition: PASS += 1 print(f" ✅ {msg}") else: FAIL += 1 print(f" ❌ {msg} — {detail}") issues.append(msg) # ═══════════════════════════════════════════ # 1. PROCESS CHECK # ═══════════════════════════════════════════ print("═══ 1. BOT PROCESSES ═══") result = subprocess.run(["ps", "aux"], capture_output=True, text=True) count = result.stdout.count("bot.py --config") ok(f"{count}/8 bots running", count >= 8, f"missing {8-count}") # check each coin individually for coin in COINS: running = f"--config {coin}.json" in result.stdout ok(f" {coin}", running, "not running") # ═══════════════════════════════════════════ # 2. DASHBOARD CHECK # ═══════════════════════════════════════════ print("═══ 2. DASHBOARD ═══") import urllib.request try: r = urllib.request.urlopen("http://localhost:5053/api/fleet", timeout=8) fleet = json.loads(r.read()) coins_found = len(fleet) ok(f"fleet API returns {coins_found} coins", coins_found >= 5, f"found {coins_found}") for coin_name in COINS.values(): present = any(v.get("label") for v in fleet.values() if v.get("label","")[1:] == coin_name) ok(f" {coin_name} in fleet", present) # check fleet HTML loads r2 = urllib.request.urlopen("http://localhost:5053/fleet", timeout=5) html = r2.read().decode() ok("fleet HTML loads", "K4LSH1_FLEET" in html) ok("sparkline charts present", "sparkline" in html) ok("gauge present", "gauge" in html) ok("donut present", "donut" in html) except Exception as e: ok("dashboard accessible", False, str(e)[:80]) # ═══════════════════════════════════════════ # 3. LOG ANALYSIS # ═══════════════════════════════════════════ print("═══ 3. LOG ERRORS ═══") log_path = HOME / "bot.log" if log_path.exists(): log = log_path.read_text() # Only count errors from last 2 hours recent = [] cutoff = time.time() - 7200 for line in log.split("\n"): try: ts_str = line[:19] # 2026-08-03 00:21:53 ts = time.mktime(time.strptime(ts_str, "%Y-%m-%d %H:%M:%S")) if ts > cutoff: recent.append(line) except Exception: continue recent_text = "\n".join(recent) if recent: last_ts = recent[-1][:19] if recent else "?" print(f" (last 2h: {len(recent)} lines, last: {last_ts})") borderlines = recent_text.count("'BORDERLINE'") loop_errors = recent_text.count("loop error") diverifier_fails = recent_text.count("diverifier failed") ok(f"BORDERLINE errors = 0", borderlines == 0, f"found {borderlines}") ok(f"loop errors = 0 ({loop_errors} found)", loop_errors < 5, f"{loop_errors} found") ok(f"diverifier failures < 20 ({diverifier_fails} found)", diverifier_fails < 30, f"{diverifier_fails}") else: ok("log file exists", False, "bot.log not found") # ═══════════════════════════════════════════ # 4. PER-COIN DB HEALTH # ═══════════════════════════════════════════ print("═══ 4. COIN DATABASES ═══") for coin in COINS: db_path = HOME / f"{coin}.db" if not db_path.exists(): ok(f"{coin} DB exists", False, "file not found") continue conn = sqlite3.connect(str(db_path)) ticks = conn.execute("SELECT COUNT(*) FROM ticks").fetchone()[0] decisions = conn.execute("SELECT COUNT(*) FROM decisions").fetchone()[0] wins = conn.execute("SELECT COUNT(*) FROM orders WHERE status='won' AND dry=0").fetchone()[0] total = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('won','lost') AND dry=0").fetchone()[0] settle = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('placed','posted') AND dry=0").fetchone()[0] stale = conn.execute("SELECT COUNT(*) FROM orders WHERE status='cancelled'").fetchone()[0] wr = (wins/total*100) if total > 0 else 0 ok(f"{coin}: {ticks} ticks", ticks > 10, f"only {ticks}") ok(f" {decisions} decisions", decisions > 0, "no decisions") ok(f" {settle} open orders", settle < 15, f"{settle} — possible stuck orders") ok(f" WR {wr:.0f}% ({wins}W/{total-wins}L)", wr > 20 or total < 5, f"WR={wr:.0f}%") conn.close() # ═══════════════════════════════════════════ # 5. DATA SOURCES # ═══════════════════════════════════════════ print("═══ 5. DATA FEEDS ═══") # Kraken spot for coin, pair in [("BTC", "XBTUSD"), ("ETH", "ETHUSD"), ("DOGE", "XDGUSD"), ("SOL", "SOLUSD")]: try: r = urllib.request.urlopen(f"https://api.kraken.com/0/public/Ticker?pair={pair}", timeout=8) data = json.loads(r.read()) ok(f"Kraken {coin}", "result" in data and len(data["result"]) > 0) except Exception: ok(f"Kraken {coin}", False, "unreachable") # Binance via proxy for coin, proxy in [("DOGE", "http://10.30.20.154:3128"), ("ETH", "http://10.30.20.189:3128")]: try: import urllib.request as ur2 ps = ur2.ProxyHandler({"https": proxy}) opener = ur2.build_opener(ps) r = opener.open(f"https://api.binance.com/api/v3/ticker/price?symbol={coin}USDT", timeout=8) data = json.loads(r.read()) ok(f"Binance {coin} via proxy", float(data["price"]) > 0) except Exception: ok(f"Binance {coin}", False, "proxy down") # Fear & Greed try: r = urllib.request.urlopen("https://api.alternative.me/fng/", timeout=8) fng = json.loads(r.read())["data"][0] ok(f"Fear&Greed: {fng['value']} ({fng['value_classification']})", True) except Exception: ok("Fear&Greed", False, "unreachable") # Kalshi balance try: sys.path.insert(0, str(HOME)) from bot import Kalshi kx = Kalshi("28d5876b-2ece-4aa3-aa17-ec96e1e706eb", str(HOME / "kalshi_private_key.pem")) b = kx.balance() bal = float(b.get("balance_dollars", 0)) ok(f"Kalshi balance: ${bal:.2f}", bal > 0) except Exception as e: ok("Kalshi balance", False, str(e)[:60]) # ═══════════════════════════════════════════ # 6. OLLAMA MODELS # ═══════════════════════════════════════════ print("═══ 6. OLLAMA INSTANCES ═══") for name, url in [("MacBook", "http://localhost:11434"), ("GamingPC", "http://10.30.20.186:11434"), ("CT518", "http://10.30.20.89:11434")]: try: r = urllib.request.urlopen(f"{url}/api/tags", timeout=8) models = json.loads(r.read()).get("models", []) ok(f"{name}: {len(models)} models loaded", len(models) > 0) except Exception: ok(f"{name}", False, "unreachable") # ═══════════════════════════════════════════ # 7. KALSHI MARKETS # ═══════════════════════════════════════════ print("═══ 7. KALSHI MARKETS ═══") for coin, series in [("DOGE","KXDOGE15M"),("SOL","KXSOL15M"),("ETH","KXETH15M")]: try: mkts = kx.markets(series, "open", 1) ok(f"{coin} 15-min market", len(mkts) > 0, "no open markets") except Exception: ok(f"{coin} 15-min market", False, "query failed") # ═══════════════════════════════════════════ # 8. GIT SYNC # ═══════════════════════════════════════════ print("═══ 8. GIT SYNC ═══") try: r = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, cwd=str(HOME)) dirty = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0 ok(f"git clean ({dirty} changed files)", dirty < 10, f"{dirty} uncommitted") except Exception: pass # ═══════════════════════════════════════════ # SUMMARY # ═══════════════════════════════════════════ print(f"\n{'═'*50}") print(f" PASS: {PASS} FAIL: {FAIL}") if issues: print(f" ISSUES:") for i in issues: print(f" ⚠ {i}") if FAIL == 0: print(" FLEET HEALTHY ✅") # update last good stamp (HOME / ".watchdog_ok").write_text(str(time.time())) else: print(f" {FAIL} ISSUE(S) DETECTED — requires attention") sys.exit(0 if FAIL == 0 else 1)