drjones/gpu-program-swapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove the duplicated starvation path; show every tenant; cache-bust assets

Browse Source 
_check_ollama_starved and _arbitrate were solving the same problem, one of them
hardcoded to two applications. The Ollama-specific version is gone and the watchdog
calls only the generic loop. The OOM retry inside switch_ollama_model no longer
purges ComfyUI by name either: it asks plan_release which tenant should give up
memory, so a third application can be the one that yields, and when the reclaim is
not enough the response names the blockers instead of implying ComfyUI was at fault.

The dashboard showed exactly two engines, which no longer matched what the service
does. A GPU Tenants panel lists every configured application ordered by priority --
VRAM held, whether it is working, whether it can be reclaimed at all, and how much it
needs -- along with the last arbitration decision and why it could or could not be
satisfied.

That panel did not appear at first, and the reason is worth fixing rather than
working around: the browser kept serving a cached app.js despite the ETag, so a
reload ran the old dashboard against the new API. Assets are now stamped with their
mtime, so a changed file is always a different URL. Anyone updating this service would
have hit the same thing.

Also verified along the way that an apparent horizontal-overflow regression was a
measurement artifact from a zero-width browser pane, not a real layout fault.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
drjones
2026-09-07 15:16:10 -07:00
parent 2b00ab4e12
commit c6455d7c6e
5 changed files with 115 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
static/app.js
View File

@@ -36,6 +36,7 @@ function updateDashboard(data) {
// Governor and arbitration state ride along in the shared snapshot.
if (data.governor) renderGovernor(data.governor);
if (data.arbitrator) renderArbitrator(data.arbitrator, data.gpu);
if (data.arbitrator) renderTenants(data.arbitrator, data.gpu);
// 1. GPU VRAM Stats
const gpu = data.gpu || {};
@@ -1135,3 +1136,56 @@ function watchChartSize() {
}
document.addEventListener('DOMContentLoaded', () => setTimeout(watchChartSize, 200));
// ---------------------------------------------------------------- tenants
function renderTenants(arb, gpu) {
const body = document.getElementById('tenants-body');
if (!body) return;
const state = arb && arb.tenant_state;
if (!state || !state.tenants) return;
const total = (gpu && gpu.vram_total_gb) || 16;
document.getElementById('tenants-free').textContent = `${state.free_gb} GB free`;
// Sorted by priority, the order arbitration actually considers them in.
const rows = [...state.tenants].sort((a, b) => b.priority - a.priority);
body.innerHTML = rows.map(t => {
const pct = Math.min((t.vram_gb / total) * 100, 100);
const bar = t.busy ? 'bg-emerald-500'
: t.reclaimable ? 'bg-cyan-600' : 'bg-amber-600';
const badge = t.busy
? '<span class="text-emerald-400">working</span>'
: t.reclaimable
? '<span class="text-slate-500">idle · reclaimable</span>'
: '<span class="text-amber-400">cannot be reclaimed</span>';
return `<div>
<div class="flex justify-between text-[11px] font-mono">
<span class="text-slate-200">${t.name}
<span class="text-slate-600">p${t.priority}</span></span>
<span class="text-slate-400">${t.vram_gb.toFixed(2)} GB · ${badge}</span>
</div>
<div class="w-full bg-slate-950 rounded-full h-1.5 mt-1 overflow-hidden border border-slate-800/60">
<div class="${bar} h-full transition-all duration-500" style="width:${pct}%"></div>
</div>
<div class="text-[10px] text-slate-600 mt-0.5">${t.reason || ''}${
t.needs_vram_gb ? ` · needs ${t.needs_vram_gb} GB to work` : ''}</div>
</div>`;
}).join('');
// The most recent arbitration decision, including why it could not be satisfied.
const dec = document.getElementById('tenants-decision');
const a = arb.last_arbitration;
if (!a) {
dec.innerHTML = '<span class="text-slate-600">No contention — nothing has needed to be released.</span>';
return;
}
const when = new Date(a.ts * 1000).toLocaleTimeString();
const blockers = (a.blockers || [])
.map(b => `${b.name} (${b.vram_gb} GB, ${b.why})`).join(', ');
dec.innerHTML =
`<span class="${a.possible ? 'text-cyan-400' : 'text-amber-400'}">${when} · ` +
`${a.demanding} short by ${a.shortfall_gb} GB</span> — ${a.reason}` +
(blockers ? `<div class="text-slate-600">blocked by: ${blockers}</div>` : '');
}

19
static/index.html
View File

@@ -680,6 +680,25 @@
<div class="grid grid-cols-1 xl:grid-cols-2 3xl:grid-cols-3 gap-5 mt-5">
<!-- All GPU tenants, however many are configured -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">
<div class="flex items-center space-x-2">
<div class="p-2 rounded-lg bg-teal-950/80 border border-teal-800 text-teal-400">
<i class="fa-solid fa-layer-group text-sm"></i>
</div>
<div>
<h3 class="font-bold text-slate-100 text-sm">GPU Tenants</h3>
<p class="text-xs text-slate-400">Every application contending for the card, from <code class="text-teal-400">tenants.json</code></p>
</div>
</div>
<span id="tenants-free" class="text-xs font-mono text-slate-500">—</span>
</div>
<div id="tenants-body" class="mt-4 space-y-2"></div>
<div id="tenants-decision" class="mt-3 text-[11px] font-mono text-slate-400"></div>
</div>
<!-- System health: makes a silently-broken dependency loud -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">