diff --git a/watchdog_test.py b/watchdog_test.py new file mode 100644 index 0000000..0677e79 --- /dev/null +++ b/watchdog_test.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""K4LSH1_OPS Watchdog — runs the same checks the cron job does every 4 hours. +Can be run manually: .venv/bin/python watchdog_test.py""" + +import json, sqlite3, subprocess, sys, time +from pathlib import Path + +HOME = Path(__file__).parent +COINS = { + "doge": "DOGE", "sol": "SOL", "eth": "ETH", "xrp": "XRP", + "btc": "BTC", "bnb": "BNB", "near": "NEAR", "zec": "ZEC", +} + +PASS = FAIL = 0 +issues = [] + +def ok(msg, condition, detail=""): + global PASS, FAIL + if condition: + PASS += 1 + print(f" ✅ {msg}") + else: + FAIL += 1 + print(f" ❌ {msg} — {detail}") + issues.append(msg) + +# ═══════════════════════════════════════════ +# 1. PROCESS CHECK +# ═══════════════════════════════════════════ +print("═══ 1. BOT PROCESSES ═══") +result = subprocess.run(["ps", "aux"], capture_output=True, text=True) +count = result.stdout.count("bot.py --config") +ok(f"{count}/8 bots running", count >= 8, f"missing {8-count}") +# check each coin individually +for coin in COINS: + running = f"--config {coin}.json" in result.stdout + ok(f" {coin}", running, "not running") + +# ═══════════════════════════════════════════ +# 2. DASHBOARD CHECK +# ═══════════════════════════════════════════ +print("═══ 2. DASHBOARD ═══") +import urllib.request +try: + r = urllib.request.urlopen("http://localhost:5053/api/fleet", timeout=8) + fleet = json.loads(r.read()) + coins_found = len(fleet) + ok(f"fleet API returns {coins_found} coins", coins_found >= 5, f"found {coins_found}") + for coin_name in COINS.values(): + present = any(v.get("label") for v in fleet.values() if v.get("label","")[1:] == coin_name) + ok(f" {coin_name} in fleet", present) + # check fleet HTML loads + r2 = urllib.request.urlopen("http://localhost:5053/fleet", timeout=5) + html = r2.read().decode() + ok("fleet HTML loads", "K4LSH1_FLEET" in html) + ok("sparkline charts present", "sparkline" in html) + ok("gauge present", "gauge" in html) + ok("donut present", "donut" in html) +except Exception as e: + ok("dashboard accessible", False, str(e)[:80]) + +# ═══════════════════════════════════════════ +# 3. LOG ANALYSIS +# ═══════════════════════════════════════════ +print("═══ 3. LOG ERRORS ═══") +log_path = HOME / "bot.log" +if log_path.exists(): + log = log_path.read_text() + # Only count errors from last 2 hours + recent = [] + cutoff = time.time() - 7200 + for line in log.split("\n"): + try: + ts_str = line[:19] # 2026-08-03 00:21:53 + ts = time.mktime(time.strptime(ts_str, "%Y-%m-%d %H:%M:%S")) + if ts > cutoff: + recent.append(line) + except Exception: + continue + recent_text = "\n".join(recent) + if recent: + last_ts = recent[-1][:19] if recent else "?" + print(f" (last 2h: {len(recent)} lines, last: {last_ts})") + borderlines = recent_text.count("'BORDERLINE'") + loop_errors = recent_text.count("loop error") + diverifier_fails = recent_text.count("diverifier failed") + ok(f"BORDERLINE errors = 0", borderlines == 0, f"found {borderlines}") + ok(f"loop errors = 0 ({loop_errors} found)", loop_errors < 5, f"{loop_errors} found") + ok(f"diverifier failures < 20 ({diverifier_fails} found)", diverifier_fails < 30, f"{diverifier_fails}") +else: + ok("log file exists", False, "bot.log not found") + +# ═══════════════════════════════════════════ +# 4. PER-COIN DB HEALTH +# ═══════════════════════════════════════════ +print("═══ 4. COIN DATABASES ═══") +for coin in COINS: + db_path = HOME / f"{coin}.db" + if not db_path.exists(): + ok(f"{coin} DB exists", False, "file not found") + continue + conn = sqlite3.connect(str(db_path)) + ticks = conn.execute("SELECT COUNT(*) FROM ticks").fetchone()[0] + decisions = conn.execute("SELECT COUNT(*) FROM decisions").fetchone()[0] + wins = conn.execute("SELECT COUNT(*) FROM orders WHERE status='won' AND dry=0").fetchone()[0] + total = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('won','lost') AND dry=0").fetchone()[0] + settle = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('placed','posted') AND dry=0").fetchone()[0] + stale = conn.execute("SELECT COUNT(*) FROM orders WHERE status='cancelled'").fetchone()[0] + wr = (wins/total*100) if total > 0 else 0 + ok(f"{coin}: {ticks} ticks", ticks > 10, f"only {ticks}") + ok(f" {decisions} decisions", decisions > 0, "no decisions") + ok(f" {settle} open orders", settle < 15, f"{settle} — possible stuck orders") + ok(f" WR {wr:.0f}% ({wins}W/{total-wins}L)", wr > 20 or total < 5, f"WR={wr:.0f}%") + conn.close() + +# ═══════════════════════════════════════════ +# 5. DATA SOURCES +# ═══════════════════════════════════════════ +print("═══ 5. DATA FEEDS ═══") +# Kraken spot +for coin, pair in [("BTC", "XBTUSD"), ("ETH", "ETHUSD"), ("DOGE", "XDGUSD"), ("SOL", "SOLUSD")]: + try: + r = urllib.request.urlopen(f"https://api.kraken.com/0/public/Ticker?pair={pair}", timeout=8) + data = json.loads(r.read()) + ok(f"Kraken {coin}", "result" in data and len(data["result"]) > 0) + except Exception: + ok(f"Kraken {coin}", False, "unreachable") + +# Binance via proxy +for coin, proxy in [("DOGE", "http://10.30.20.154:3128"), ("ETH", "http://10.30.20.189:3128")]: + try: + import urllib.request as ur2 + ps = ur2.ProxyHandler({"https": proxy}) + opener = ur2.build_opener(ps) + r = opener.open(f"https://api.binance.com/api/v3/ticker/price?symbol={coin}USDT", timeout=8) + data = json.loads(r.read()) + ok(f"Binance {coin} via proxy", float(data["price"]) > 0) + except Exception: + ok(f"Binance {coin}", False, "proxy down") + +# Fear & Greed +try: + r = urllib.request.urlopen("https://api.alternative.me/fng/", timeout=8) + fng = json.loads(r.read())["data"][0] + ok(f"Fear&Greed: {fng['value']} ({fng['value_classification']})", True) +except Exception: + ok("Fear&Greed", False, "unreachable") + +# Kalshi balance +try: + sys.path.insert(0, str(HOME)) + from bot import Kalshi + kx = Kalshi("28d5876b-2ece-4aa3-aa17-ec96e1e706eb", str(HOME / "kalshi_private_key.pem")) + b = kx.balance() + bal = float(b.get("balance_dollars", 0)) + ok(f"Kalshi balance: ${bal:.2f}", bal > 0) +except Exception as e: + ok("Kalshi balance", False, str(e)[:60]) + +# ═══════════════════════════════════════════ +# 6. OLLAMA MODELS +# ═══════════════════════════════════════════ +print("═══ 6. OLLAMA INSTANCES ═══") +for name, url in [("MacBook", "http://localhost:11434"), ("GamingPC", "http://10.30.20.186:11434"), ("CT518", "http://10.30.20.89:11434")]: + try: + r = urllib.request.urlopen(f"{url}/api/tags", timeout=8) + models = json.loads(r.read()).get("models", []) + ok(f"{name}: {len(models)} models loaded", len(models) > 0) + except Exception: + ok(f"{name}", False, "unreachable") + +# ═══════════════════════════════════════════ +# 7. KALSHI MARKETS +# ═══════════════════════════════════════════ +print("═══ 7. KALSHI MARKETS ═══") +for coin, series in [("DOGE","KXDOGE15M"),("SOL","KXSOL15M"),("ETH","KXETH15M")]: + try: + mkts = kx.markets(series, "open", 1) + ok(f"{coin} 15-min market", len(mkts) > 0, "no open markets") + except Exception: + ok(f"{coin} 15-min market", False, "query failed") + +# ═══════════════════════════════════════════ +# 8. GIT SYNC +# ═══════════════════════════════════════════ +print("═══ 8. GIT SYNC ═══") +try: + r = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, cwd=str(HOME)) + dirty = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0 + ok(f"git clean ({dirty} changed files)", dirty < 10, f"{dirty} uncommitted") +except Exception: + pass + +# ═══════════════════════════════════════════ +# SUMMARY +# ═══════════════════════════════════════════ +print(f"\n{'═'*50}") +print(f" PASS: {PASS} FAIL: {FAIL}") +if issues: + print(f" ISSUES:") + for i in issues: + print(f" ⚠ {i}") +if FAIL == 0: + print(" FLEET HEALTHY ✅") + # update last good stamp + (HOME / ".watchdog_ok").write_text(str(time.time())) +else: + print(f" {FAIL} ISSUE(S) DETECTED — requires attention") + +sys.exit(0 if FAIL == 0 else 1)