drjones/gpu-program-swapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a dependency self-check; cut the SSE payload by 74%

Browse Source 
Self-check. Fan control failed for an entire session -- recoverably, and completely
invisibly. It appeared once, inside one field of one log line, and nothing ever
asked whether fan control worked. health.py now checks everything this service
depends on (NVML, passwordless sudo for nvidia-smi, fan control via the headless X
server, overclock drift, the telemetry store, residency measurement capability,
model directories, the ComfyUI websocket, and both upstream HTTP services) and
reports for each one what is broken, what that breaks, and how to fix it. Exposed at
GET /api/health, as an MCP tool, and as a dashboard panel that collapses to a badge
when healthy and expands to impact-and-fix when not. Current state: 9 ok, 1 degraded
(the known cachestat permission limit on Ollama's blobs).

A self-check that returns ok while a dependency is broken is worse than none, so the
tests drive each check to its failure state -- including the exact "Error resolving
target specification 'gpu:0'" string from the original incident -- and assert that a
check which raises surfaces as failed rather than taking down the endpoint.

SSE payload. The installed-model catalog was 10.6 KB of a 13.1 KB frame, 81% of the
stream, re-sent to every subscriber every second despite changing only when a model
is pulled or removed: 135 MB/hour across three tabs. It is now sent on a
subscriber's first frame and whenever the set changes; the client keeps the last
known list. Steady-state frames dropped from 14041 to 3664 bytes, a 74% reduction,
and /api/stats still returns the complete snapshot for API consumers.

Tests: 182 (was 169).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
drjones
2026-09-05 19:15:52 -07:00
parent 25b601e24a
commit fb104ac9c0
6 changed files with 437 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
static/app.js
View File

@@ -177,9 +177,13 @@ function updateDashboard(data) {
document.getElementById('ollama-context').textContent = 'Idle';
}
// The catalog is omitted from stream frames when unchanged, so keep the last one.
if (ollama.installed_models && ollama.installed_models.length > 0) {
document.getElementById('ollama-total-models').textContent = ollama.installed_models.length;
updateModelSelect(ollama.installed_models, ollama.active_model_name);
currentInstalledModels = ollama.installed_models;
}
if (currentInstalledModels && currentInstalledModels.length > 0) {
document.getElementById('ollama-total-models').textContent = currentInstalledModels.length;
updateModelSelect(currentInstalledModels, ollama.active_model_name);
}
} else {
document.getElementById('ollama-status-text').textContent = 'OFFLINE';
@@ -927,3 +931,48 @@ function renderArbitrator(arb, gpu) {
}
el('arb-backoff').textContent = parts.join(' · ');
}
// ---------------------------------------------------------------- health
async function fetchHealth(verbose = false) {
const badge = document.getElementById('health-badge');
const body = document.getElementById('health-body');
if (!badge) return;
try {
const d = await (await fetch('/api/health')).json();
const style = {
ok: 'bg-emerald-950/70 border border-emerald-800 text-emerald-300',
degraded: 'bg-amber-950/70 border border-amber-800 text-amber-300',
failed: 'bg-rose-950/70 border border-rose-800 text-rose-300',
}[d.status];
badge.className = `px-2 py-1 text-xs font-bold rounded-lg ${style}`;
badge.textContent = `${d.status.toUpperCase()} · ${d.summary}`;
// Healthy checks collapse to one line; anything wrong is shown in full with the
// impact and the fix, because that is the information you actually need.
const problems = d.checks.filter(c => c.status !== 'ok');
const shown = (verbose || problems.length) ? (verbose ? d.checks : problems) : [];
body.innerHTML = shown.map(c => {
const colour = c.status === 'ok' ? 'text-emerald-400'
: c.status === 'degraded' ? 'text-amber-400' : 'text-rose-400';
const mark = c.status === 'ok' ? '&#10003;' : c.status === 'degraded' ? '!' : '&#10007;';
let html = `<div><span class="${colour}">${mark} ${c.name}</span>` +
`<span class="text-slate-400"> — ${c.detail}</span></div>`;
if (c.status !== 'ok') {
if (c.impact) html += `<div class="text-slate-500 pl-4">impact: ${c.impact}</div>`;
if (c.fix) html += `<div class="text-slate-500 pl-4">fix: ${c.fix}</div>`;
}
return html;
}).join('') || '<div class="text-emerald-500">All dependencies healthy.</div>';
} catch (e) {
badge.className = 'px-2 py-1 text-xs font-bold rounded-lg bg-rose-950/70 border border-rose-800 text-rose-300';
badge.textContent = 'UNREACHABLE';
body.innerHTML = `<div class="text-rose-400">${e}</div>`;
}
}
document.addEventListener('DOMContentLoaded', () => {
fetchHealth(false);
setInterval(() => fetchHealth(false), 60000);
});

20
static/index.html
View File

@@ -669,6 +669,26 @@
<div class="grid grid-cols-1 xl:grid-cols-2 gap-5 mt-5">
<!-- System health: makes a silently-broken dependency loud -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">
<div class="flex items-center space-x-2">
<div class="p-2 rounded-lg bg-sky-950/80 border border-sky-800 text-sky-400">
<i class="fa-solid fa-stethoscope text-sm"></i>
</div>
<div>
<h3 class="font-bold text-slate-100 text-sm">System Health</h3>
<p class="text-xs text-slate-400">Every dependency, with impact and how to fix it</p>
</div>
</div>
<div class="flex items-center space-x-2">
<span id="health-badge" class="px-2 py-1 text-xs font-bold rounded-lg bg-slate-800 text-slate-400">checking…</span>
<button onclick="fetchHealth(true)" class="px-2.5 py-1 text-xs font-semibold rounded-lg bg-slate-800 border border-slate-700 text-slate-300 hover:bg-slate-700 transition">Recheck</button>
</div>
</div>
<div id="health-body" class="mt-4 space-y-1.5 text-[11px] font-mono"></div>
</div>
<!-- VRAM Arbitration — the core handoff, previously invisible -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">