watchdog test script + log filter fix

watchdog_test.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""K4LSH1_OPS Watchdog — runs the same checks the cron job does every 4 hours.
+Can be run manually: .venv/bin/python watchdog_test.py"""
+
+import json, sqlite3, subprocess, sys, time
+from pathlib import Path
+
+HOME = Path(__file__).parent
+COINS = {
+    "doge": "DOGE", "sol": "SOL", "eth": "ETH", "xrp": "XRP",
+    "btc": "BTC", "bnb": "BNB", "near": "NEAR", "zec": "ZEC",
+}
+
+PASS = FAIL = 0
+issues = []
+
+def ok(msg, condition, detail=""):
+    global PASS, FAIL
+    if condition:
+        PASS += 1
+        print(f"  ✅ {msg}")
+    else:
+        FAIL += 1
+        print(f"  ❌ {msg}  — {detail}")
+        issues.append(msg)
+
+# ═══════════════════════════════════════════
+# 1. PROCESS CHECK
+# ═══════════════════════════════════════════
+print("═══ 1. BOT PROCESSES ═══")
+result = subprocess.run(["ps", "aux"], capture_output=True, text=True)
+count = result.stdout.count("bot.py --config")
+ok(f"{count}/8 bots running", count >= 8, f"missing {8-count}")
+# check each coin individually
+for coin in COINS:
+    running = f"--config {coin}.json" in result.stdout
+    ok(f"  {coin}", running, "not running")
+
+# ═══════════════════════════════════════════
+# 2. DASHBOARD CHECK
+# ═══════════════════════════════════════════
+print("═══ 2. DASHBOARD ═══")
+import urllib.request
+try:
+    r = urllib.request.urlopen("http://localhost:5053/api/fleet", timeout=8)
+    fleet = json.loads(r.read())
+    coins_found = len(fleet)
+    ok(f"fleet API returns {coins_found} coins", coins_found >= 5, f"found {coins_found}")
+    for coin_name in COINS.values():
+        present = any(v.get("label") for v in fleet.values() if v.get("label","")[1:] == coin_name)
+        ok(f"  {coin_name} in fleet", present)
+    # check fleet HTML loads
+    r2 = urllib.request.urlopen("http://localhost:5053/fleet", timeout=5)
+    html = r2.read().decode()
+    ok("fleet HTML loads", "K4LSH1_FLEET" in html)
+    ok("sparkline charts present", "sparkline" in html)
+    ok("gauge present", "gauge" in html)
+    ok("donut present", "donut" in html)
+except Exception as e:
+    ok("dashboard accessible", False, str(e)[:80])
+
+# ═══════════════════════════════════════════
+# 3. LOG ANALYSIS
+# ═══════════════════════════════════════════
+print("═══ 3. LOG ERRORS ═══")
+log_path = HOME / "bot.log"
+if log_path.exists():
+    log = log_path.read_text()
+    # Only count errors from last 2 hours
+    recent = []
+    cutoff = time.time() - 7200
+    for line in log.split("\n"):
+        try:
+            ts_str = line[:19]  # 2026-08-03 00:21:53
+            ts = time.mktime(time.strptime(ts_str, "%Y-%m-%d %H:%M:%S"))
+            if ts > cutoff:
+                recent.append(line)
+        except Exception:
+            continue
+    recent_text = "\n".join(recent)
+    if recent:
+        last_ts = recent[-1][:19] if recent else "?"
+        print(f"  (last 2h: {len(recent)} lines, last: {last_ts})")
+    borderlines = recent_text.count("'BORDERLINE'")
+    loop_errors = recent_text.count("loop error")
+    diverifier_fails = recent_text.count("diverifier failed")
+    ok(f"BORDERLINE errors = 0", borderlines == 0, f"found {borderlines}")
+    ok(f"loop errors = 0 ({loop_errors} found)", loop_errors < 5, f"{loop_errors} found")
+    ok(f"diverifier failures < 20 ({diverifier_fails} found)", diverifier_fails < 30, f"{diverifier_fails}")
+else:
+    ok("log file exists", False, "bot.log not found")
+
+# ═══════════════════════════════════════════
+# 4. PER-COIN DB HEALTH
+# ═══════════════════════════════════════════
+print("═══ 4. COIN DATABASES ═══")
+for coin in COINS:
+    db_path = HOME / f"{coin}.db"
+    if not db_path.exists():
+        ok(f"{coin} DB exists", False, "file not found")
+        continue
+    conn = sqlite3.connect(str(db_path))
+    ticks = conn.execute("SELECT COUNT(*) FROM ticks").fetchone()[0]
+    decisions = conn.execute("SELECT COUNT(*) FROM decisions").fetchone()[0]
+    wins = conn.execute("SELECT COUNT(*) FROM orders WHERE status='won' AND dry=0").fetchone()[0]
+    total = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('won','lost') AND dry=0").fetchone()[0]
+    settle = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('placed','posted') AND dry=0").fetchone()[0]
+    stale = conn.execute("SELECT COUNT(*) FROM orders WHERE status='cancelled'").fetchone()[0]
+    wr = (wins/total*100) if total > 0 else 0
+    ok(f"{coin}: {ticks} ticks", ticks > 10, f"only {ticks}")
+    ok(f"  {decisions} decisions", decisions > 0, "no decisions")
+    ok(f"  {settle} open orders", settle < 15, f"{settle} — possible stuck orders")
+    ok(f"  WR {wr:.0f}% ({wins}W/{total-wins}L)", wr > 20 or total < 5, f"WR={wr:.0f}%")
+    conn.close()
+
+# ═══════════════════════════════════════════
+# 5. DATA SOURCES
+# ═══════════════════════════════════════════
+print("═══ 5. DATA FEEDS ═══")
+# Kraken spot
+for coin, pair in [("BTC", "XBTUSD"), ("ETH", "ETHUSD"), ("DOGE", "XDGUSD"), ("SOL", "SOLUSD")]:
+    try:
+        r = urllib.request.urlopen(f"https://api.kraken.com/0/public/Ticker?pair={pair}", timeout=8)
+        data = json.loads(r.read())
+        ok(f"Kraken {coin}", "result" in data and len(data["result"]) > 0)
+    except Exception:
+        ok(f"Kraken {coin}", False, "unreachable")
+
+# Binance via proxy
+for coin, proxy in [("DOGE", "http://10.30.20.154:3128"), ("ETH", "http://10.30.20.189:3128")]:
+    try:
+        import urllib.request as ur2
+        ps = ur2.ProxyHandler({"https": proxy})
+        opener = ur2.build_opener(ps)
+        r = opener.open(f"https://api.binance.com/api/v3/ticker/price?symbol={coin}USDT", timeout=8)
+        data = json.loads(r.read())
+        ok(f"Binance {coin} via proxy", float(data["price"]) > 0)
+    except Exception:
+        ok(f"Binance {coin}", False, "proxy down")
+
+# Fear & Greed
+try:
+    r = urllib.request.urlopen("https://api.alternative.me/fng/", timeout=8)
+    fng = json.loads(r.read())["data"][0]
+    ok(f"Fear&Greed: {fng['value']} ({fng['value_classification']})", True)
+except Exception:
+    ok("Fear&Greed", False, "unreachable")
+
+# Kalshi balance
+try:
+    sys.path.insert(0, str(HOME))
+    from bot import Kalshi
+    kx = Kalshi("28d5876b-2ece-4aa3-aa17-ec96e1e706eb", str(HOME / "kalshi_private_key.pem"))
+    b = kx.balance()
+    bal = float(b.get("balance_dollars", 0))
+    ok(f"Kalshi balance: ${bal:.2f}", bal > 0)
+except Exception as e:
+    ok("Kalshi balance", False, str(e)[:60])
+
+# ═══════════════════════════════════════════
+# 6. OLLAMA MODELS
+# ═══════════════════════════════════════════
+print("═══ 6. OLLAMA INSTANCES ═══")
+for name, url in [("MacBook", "http://localhost:11434"), ("GamingPC", "http://10.30.20.186:11434"), ("CT518", "http://10.30.20.89:11434")]:
+    try:
+        r = urllib.request.urlopen(f"{url}/api/tags", timeout=8)
+        models = json.loads(r.read()).get("models", [])
+        ok(f"{name}: {len(models)} models loaded", len(models) > 0)
+    except Exception:
+        ok(f"{name}", False, "unreachable")
+
+# ═══════════════════════════════════════════
+# 7. KALSHI MARKETS
+# ═══════════════════════════════════════════
+print("═══ 7. KALSHI MARKETS ═══")
+for coin, series in [("DOGE","KXDOGE15M"),("SOL","KXSOL15M"),("ETH","KXETH15M")]:
+    try:
+        mkts = kx.markets(series, "open", 1)
+        ok(f"{coin} 15-min market", len(mkts) > 0, "no open markets")
+    except Exception:
+        ok(f"{coin} 15-min market", False, "query failed")
+
+# ═══════════════════════════════════════════
+# 8. GIT SYNC
+# ═══════════════════════════════════════════
+print("═══ 8. GIT SYNC ═══")
+try:
+    r = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, cwd=str(HOME))
+    dirty = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0
+    ok(f"git clean ({dirty} changed files)", dirty < 10, f"{dirty} uncommitted")
+except Exception:
+    pass
+
+# ═══════════════════════════════════════════
+# SUMMARY
+# ═══════════════════════════════════════════
+print(f"\n{'═'*50}")
+print(f"  PASS: {PASS}  FAIL: {FAIL}")
+if issues:
+    print(f"  ISSUES:")
+    for i in issues:
+        print(f"    ⚠ {i}")
+if FAIL == 0:
+    print("  FLEET HEALTHY ✅")
+    # update last good stamp
+    (HOME / ".watchdog_ok").write_text(str(time.time()))
+else:
+    print(f"  {FAIL} ISSUE(S) DETECTED — requires attention")
+
+sys.exit(0 if FAIL == 0 else 1)