drjones/gpu-program-swapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test the job queue, and surface it in the dashboard and MCP

Browse Source 
The queue shipped with no tests despite having produced four bugs during
development, which is the wrong order. 21 tests now cover the parts whose failure
modes are not obvious from reading the code: ordering by priority then FIFO, that
depth is genuinely unbounded and survives a restart, that running work is never
cancelled, that a job abandoned mid-run is requeued rather than left RUNNING
forever, and that an LLM job's VRAM requirement comes from its own model rather
than a tenant-wide figure -- the mistake that dispatched a 14.9 GB model into 8 GB
of free memory and killed llama-server three times.

The dashboard had no view of the queue at all, so a stuck queue was
indistinguishable from an empty one. The Job Queue panel shows what is running and
what it released to get there, pending jobs in execution order with their wait time
and a cancel control, and -- when the scheduler is blocked -- how long it has been
waiting and why.

MCP gains queue_job, get_job_queue and cancel_job, so an agent can line work up
rather than firing a request and hoping the GPU is free.

Tests: 271.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
drjones
2026-09-07 17:41:55 -07:00
parent 0fbc3963b9
commit 48fbde040c
5 changed files with 310 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
static/app.js
View File

@@ -1189,3 +1189,74 @@ function renderTenants(arb, gpu) {
`${a.demanding} short by ${a.shortfall_gb} GB</span> — ${a.reason}` +
(blockers ? `<div class="text-slate-600">blocked by: ${blockers}</div>` : '');
}
// ---------------------------------------------------------------- job queue
async function fetchQueue() {
const body = document.getElementById('queue-body');
if (!body) return;
try {
const d = await (await fetch('/api/jobs?limit=40')).json();
const s = d.scheduler || {};
document.getElementById('queue-summary').textContent =
`${s.queue_depth ?? 0} queued · ${s.completed ?? 0} done · ${s.failed ?? 0} failed`;
// What the scheduler is doing right now, including why it is waiting. A job that
// cannot get VRAM used to sit silent, which made a stuck queue indistinguishable
// from an empty one.
const cur = document.getElementById('queue-current');
if (s.current) {
cur.innerHTML = `<span class="text-emerald-400">running</span> ` +
`<span class="text-slate-200">${s.current.tenant}</span>` +
`<span class="text-slate-500"> · ${s.current.label || s.current.id}</span>` +
(s.current.room && s.current.room.released && s.current.room.released.length
? `<span class="text-cyan-400"> · released ${s.current.room.released.join(', ')}</span>` : '');
} else if (s.blocked) {
cur.innerHTML = `<span class="text-amber-400">waiting ${s.blocked.waited_s}s</span> ` +
`<span class="text-slate-200">${s.blocked.tenant}</span>` +
`<span class="text-slate-500"> · ${s.blocked.reason || ''}</span>`;
} else {
cur.innerHTML = '<span class="text-slate-600">scheduler idle</span>';
}
const colour = {
pending: 'text-slate-400', running: 'text-emerald-400', done: 'text-cyan-500',
failed: 'text-rose-400', cancelled: 'text-slate-600',
};
body.innerHTML = (d.jobs || []).map(job => {
const when = job.state === 'pending'
? `waiting ${job.waiting_s}s`
: (job.duration_s != null ? `${job.duration_s}s` : '');
return `<div class="flex justify-between text-[11px] font-mono gap-2">
<span class="truncate">
<span class="${colour[job.state] || 'text-slate-400'}">${job.state}</span>
<span class="text-slate-600"> p${job.priority}</span>
<span class="text-slate-200"> ${job.tenant}</span>
<span class="text-slate-500">${job.label ? ' · ' + job.label : ''}</span>
</span>
<span class="text-slate-500 whitespace-nowrap">${when}${
job.state === 'pending'
? ` <button onclick="cancelJob('${job.id}')" class="text-rose-500 hover:text-rose-300 ml-1">cancel</button>`
: ''}</span>
</div>${job.error ? `<div class="text-[10px] text-rose-500/80 pl-2 truncate">${job.error}</div>` : ''}`;
}).join('') || '<div class="text-slate-600 text-[11px]">No jobs yet.</div>';
} catch (e) {
body.innerHTML = `<div class="text-rose-400 text-[11px]">${e}</div>`;
}
}
async function cancelJob(id) {
await fetch(`/api/jobs/${id}`, { method: 'DELETE' });
fetchQueue();
}
async function clearQueue() {
await fetch('/api/jobs', { method: 'DELETE' });
fetchQueue();
}
document.addEventListener('DOMContentLoaded', () => {
fetchQueue();
setInterval(fetchQueue, 2000);
});

22
static/index.html
View File

@@ -681,6 +681,28 @@
<!-- The cross-application job queue -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">
<div class="flex items-center space-x-2">
<div class="p-2 rounded-lg bg-violet-950/80 border border-violet-800 text-violet-400">
<i class="fa-solid fa-list-check text-sm"></i>
</div>
<div>
<h3 class="font-bold text-slate-100 text-sm">Job Queue</h3>
<p class="text-xs text-slate-400">Work lined up across every application, highest priority first</p>
</div>
</div>
<div class="flex items-center space-x-2">
<span id="queue-summary" class="text-xs font-mono text-slate-500">—</span>
<button onclick="clearQueue()" class="px-2.5 py-1 text-xs font-semibold rounded-lg bg-slate-800 border border-slate-700 text-slate-300 hover:bg-slate-700 transition">Clear pending</button>
</div>
</div>
<div id="queue-current" class="mt-4 text-[11px] font-mono"></div>
<div id="queue-body" class="mt-3 space-y-1.5 max-h-64 overflow-y-auto pr-1"></div>
</div>
<!-- All GPU tenants, however many are configured -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 xl:col-span-2">
<div class="flex items-center justify-between pb-3 border-b border-slate-800">