watchdog test script + log filter fix
This commit is contained in:
drjones
210
watchdog_test.py
Normal file
210
watchdog_test.py
Normal file
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""K4LSH1_OPS Watchdog — runs the same checks the cron job does every 4 hours.
|
||||
Can be run manually: .venv/bin/python watchdog_test.py"""
|
||||
|
||||
import json, sqlite3, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
|
||||
HOME = Path(__file__).parent
|
||||
COINS = {
|
||||
"doge": "DOGE", "sol": "SOL", "eth": "ETH", "xrp": "XRP",
|
||||
"btc": "BTC", "bnb": "BNB", "near": "NEAR", "zec": "ZEC",
|
||||
}
|
||||
|
||||
PASS = FAIL = 0
|
||||
issues = []
|
||||
|
||||
def ok(msg, condition, detail=""):
|
||||
global PASS, FAIL
|
||||
if condition:
|
||||
PASS += 1
|
||||
print(f" ✅ {msg}")
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f" ❌ {msg} — {detail}")
|
||||
issues.append(msg)
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 1. PROCESS CHECK
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 1. BOT PROCESSES ═══")
|
||||
result = subprocess.run(["ps", "aux"], capture_output=True, text=True)
|
||||
count = result.stdout.count("bot.py --config")
|
||||
ok(f"{count}/8 bots running", count >= 8, f"missing {8-count}")
|
||||
# check each coin individually
|
||||
for coin in COINS:
|
||||
running = f"--config {coin}.json" in result.stdout
|
||||
ok(f" {coin}", running, "not running")
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 2. DASHBOARD CHECK
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 2. DASHBOARD ═══")
|
||||
import urllib.request
|
||||
try:
|
||||
r = urllib.request.urlopen("http://localhost:5053/api/fleet", timeout=8)
|
||||
fleet = json.loads(r.read())
|
||||
coins_found = len(fleet)
|
||||
ok(f"fleet API returns {coins_found} coins", coins_found >= 5, f"found {coins_found}")
|
||||
for coin_name in COINS.values():
|
||||
present = any(v.get("label") for v in fleet.values() if v.get("label","")[1:] == coin_name)
|
||||
ok(f" {coin_name} in fleet", present)
|
||||
# check fleet HTML loads
|
||||
r2 = urllib.request.urlopen("http://localhost:5053/fleet", timeout=5)
|
||||
html = r2.read().decode()
|
||||
ok("fleet HTML loads", "K4LSH1_FLEET" in html)
|
||||
ok("sparkline charts present", "sparkline" in html)
|
||||
ok("gauge present", "gauge" in html)
|
||||
ok("donut present", "donut" in html)
|
||||
except Exception as e:
|
||||
ok("dashboard accessible", False, str(e)[:80])
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 3. LOG ANALYSIS
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 3. LOG ERRORS ═══")
|
||||
log_path = HOME / "bot.log"
|
||||
if log_path.exists():
|
||||
log = log_path.read_text()
|
||||
# Only count errors from last 2 hours
|
||||
recent = []
|
||||
cutoff = time.time() - 7200
|
||||
for line in log.split("\n"):
|
||||
try:
|
||||
ts_str = line[:19] # 2026-08-03 00:21:53
|
||||
ts = time.mktime(time.strptime(ts_str, "%Y-%m-%d %H:%M:%S"))
|
||||
if ts > cutoff:
|
||||
recent.append(line)
|
||||
except Exception:
|
||||
continue
|
||||
recent_text = "\n".join(recent)
|
||||
if recent:
|
||||
last_ts = recent[-1][:19] if recent else "?"
|
||||
print(f" (last 2h: {len(recent)} lines, last: {last_ts})")
|
||||
borderlines = recent_text.count("'BORDERLINE'")
|
||||
loop_errors = recent_text.count("loop error")
|
||||
diverifier_fails = recent_text.count("diverifier failed")
|
||||
ok(f"BORDERLINE errors = 0", borderlines == 0, f"found {borderlines}")
|
||||
ok(f"loop errors = 0 ({loop_errors} found)", loop_errors < 5, f"{loop_errors} found")
|
||||
ok(f"diverifier failures < 20 ({diverifier_fails} found)", diverifier_fails < 30, f"{diverifier_fails}")
|
||||
else:
|
||||
ok("log file exists", False, "bot.log not found")
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 4. PER-COIN DB HEALTH
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 4. COIN DATABASES ═══")
|
||||
for coin in COINS:
|
||||
db_path = HOME / f"{coin}.db"
|
||||
if not db_path.exists():
|
||||
ok(f"{coin} DB exists", False, "file not found")
|
||||
continue
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
ticks = conn.execute("SELECT COUNT(*) FROM ticks").fetchone()[0]
|
||||
decisions = conn.execute("SELECT COUNT(*) FROM decisions").fetchone()[0]
|
||||
wins = conn.execute("SELECT COUNT(*) FROM orders WHERE status='won' AND dry=0").fetchone()[0]
|
||||
total = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('won','lost') AND dry=0").fetchone()[0]
|
||||
settle = conn.execute("SELECT COUNT(*) FROM orders WHERE status IN ('placed','posted') AND dry=0").fetchone()[0]
|
||||
stale = conn.execute("SELECT COUNT(*) FROM orders WHERE status='cancelled'").fetchone()[0]
|
||||
wr = (wins/total*100) if total > 0 else 0
|
||||
ok(f"{coin}: {ticks} ticks", ticks > 10, f"only {ticks}")
|
||||
ok(f" {decisions} decisions", decisions > 0, "no decisions")
|
||||
ok(f" {settle} open orders", settle < 15, f"{settle} — possible stuck orders")
|
||||
ok(f" WR {wr:.0f}% ({wins}W/{total-wins}L)", wr > 20 or total < 5, f"WR={wr:.0f}%")
|
||||
conn.close()
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 5. DATA SOURCES
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 5. DATA FEEDS ═══")
|
||||
# Kraken spot
|
||||
for coin, pair in [("BTC", "XBTUSD"), ("ETH", "ETHUSD"), ("DOGE", "XDGUSD"), ("SOL", "SOLUSD")]:
|
||||
try:
|
||||
r = urllib.request.urlopen(f"https://api.kraken.com/0/public/Ticker?pair={pair}", timeout=8)
|
||||
data = json.loads(r.read())
|
||||
ok(f"Kraken {coin}", "result" in data and len(data["result"]) > 0)
|
||||
except Exception:
|
||||
ok(f"Kraken {coin}", False, "unreachable")
|
||||
|
||||
# Binance via proxy
|
||||
for coin, proxy in [("DOGE", "http://10.30.20.154:3128"), ("ETH", "http://10.30.20.189:3128")]:
|
||||
try:
|
||||
import urllib.request as ur2
|
||||
ps = ur2.ProxyHandler({"https": proxy})
|
||||
opener = ur2.build_opener(ps)
|
||||
r = opener.open(f"https://api.binance.com/api/v3/ticker/price?symbol={coin}USDT", timeout=8)
|
||||
data = json.loads(r.read())
|
||||
ok(f"Binance {coin} via proxy", float(data["price"]) > 0)
|
||||
except Exception:
|
||||
ok(f"Binance {coin}", False, "proxy down")
|
||||
|
||||
# Fear & Greed
|
||||
try:
|
||||
r = urllib.request.urlopen("https://api.alternative.me/fng/", timeout=8)
|
||||
fng = json.loads(r.read())["data"][0]
|
||||
ok(f"Fear&Greed: {fng['value']} ({fng['value_classification']})", True)
|
||||
except Exception:
|
||||
ok("Fear&Greed", False, "unreachable")
|
||||
|
||||
# Kalshi balance
|
||||
try:
|
||||
sys.path.insert(0, str(HOME))
|
||||
from bot import Kalshi
|
||||
kx = Kalshi("28d5876b-2ece-4aa3-aa17-ec96e1e706eb", str(HOME / "kalshi_private_key.pem"))
|
||||
b = kx.balance()
|
||||
bal = float(b.get("balance_dollars", 0))
|
||||
ok(f"Kalshi balance: ${bal:.2f}", bal > 0)
|
||||
except Exception as e:
|
||||
ok("Kalshi balance", False, str(e)[:60])
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 6. OLLAMA MODELS
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 6. OLLAMA INSTANCES ═══")
|
||||
for name, url in [("MacBook", "http://localhost:11434"), ("GamingPC", "http://10.30.20.186:11434"), ("CT518", "http://10.30.20.89:11434")]:
|
||||
try:
|
||||
r = urllib.request.urlopen(f"{url}/api/tags", timeout=8)
|
||||
models = json.loads(r.read()).get("models", [])
|
||||
ok(f"{name}: {len(models)} models loaded", len(models) > 0)
|
||||
except Exception:
|
||||
ok(f"{name}", False, "unreachable")
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 7. KALSHI MARKETS
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 7. KALSHI MARKETS ═══")
|
||||
for coin, series in [("DOGE","KXDOGE15M"),("SOL","KXSOL15M"),("ETH","KXETH15M")]:
|
||||
try:
|
||||
mkts = kx.markets(series, "open", 1)
|
||||
ok(f"{coin} 15-min market", len(mkts) > 0, "no open markets")
|
||||
except Exception:
|
||||
ok(f"{coin} 15-min market", False, "query failed")
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# 8. GIT SYNC
|
||||
# ═══════════════════════════════════════════
|
||||
print("═══ 8. GIT SYNC ═══")
|
||||
try:
|
||||
r = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, cwd=str(HOME))
|
||||
dirty = len(r.stdout.strip().split("\n")) if r.stdout.strip() else 0
|
||||
ok(f"git clean ({dirty} changed files)", dirty < 10, f"{dirty} uncommitted")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ═══════════════════════════════════════════
|
||||
print(f"\n{'═'*50}")
|
||||
print(f" PASS: {PASS} FAIL: {FAIL}")
|
||||
if issues:
|
||||
print(f" ISSUES:")
|
||||
for i in issues:
|
||||
print(f" ⚠ {i}")
|
||||
if FAIL == 0:
|
||||
print(" FLEET HEALTHY ✅")
|
||||
# update last good stamp
|
||||
(HOME / ".watchdog_ok").write_text(str(time.time()))
|
||||
else:
|
||||
print(f" {FAIL} ISSUE(S) DETECTED — requires attention")
|
||||
|
||||
sys.exit(0 if FAIL == 0 else 1)
|
||||
Reference in New Issue
Block a user