bacaf50713
The persisted counters showed 19 timeouts in 20 yields. All 19 were one model, ornith-1.5:9b-cron, in two bursts at 06:25 and 06:33. Telemetry for that window shows the GPU pinned at 96-97% with Ollama holding 14.92 GB throughout: the model was mid-generation. Ollama will not unload a model that is inferencing, so every request failed, and with a 1s trigger debounce against a 10s blocking wait the arbitrator simply asked again, four times per burst, blocking the loop for 40s. Ollama's behaviour is correct. Ours was wrong in three ways. Busy is now a distinct outcome. _await_vram_release returns "released", "busy" or "stuck": VRAM that has not moved while the GPU is pinned means a generation is in flight, which is not a failure. With OLLAMA_NUM_PARALLEL=1 our keep_alive:0 request queues behind the running one and applies the moment it finishes, so the correct response is to stop waiting, not to retry. Only "stuck" -- VRAM held with an idle GPU -- is a real fault. The wait is short again (2s, from 10s) because blocking helps nobody: ComfyUI is not gated on our return value, and every blocked second stalls the watchdog and profile switching. Callers who genuinely want to wait out an inference can pass wait_for_generation=true. The unload POST itself now gets a 120s client timeout, since a 5s one could drop the connection before Ollama ever processed a request queued behind a long generation, losing the unload entirely. A busy model gets per-model backoff (5s, 15s, 30s, 60s) instead of being asked again every second, and a detached watcher confirms and logs the release when the generation ends, so the event log tells the whole story rather than stopping at "deferred". Measured: a mid-generation yield now returns busy in 610ms instead of blocking 10s, and the queued unload lands on its own 3s later when the generation completes. Counters are honest: yields (released), yield_deferred_busy, deferred_releases, yield_stalled. The old yield_timeouts conflated a healthy cron job with a fault and implied a 95% failure rate. Also adds a VRAM Arbitration panel to the dashboard. The arbitrator is the core of this application and its state was not displayed anywhere -- there was no way to see whether handoffs were working, which is why this went unnoticed until the persisted counters were read by hand. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
918 lines
42 KiB
JavaScript
918 lines
42 KiB
JavaScript
let evtSource = null;
|
|
let currentInstalledModels = [];
|
|
|
|
// Initialize SSE Stream
|
|
function initSSE() {
|
|
if (evtSource) {
|
|
evtSource.close();
|
|
}
|
|
|
|
evtSource = new EventSource('/api/stream');
|
|
|
|
evtSource.onopen = () => {
|
|
document.getElementById('sse-badge').textContent = 'SSE LIVE';
|
|
document.getElementById('sse-badge').className = 'text-emerald-400 font-bold';
|
|
};
|
|
|
|
evtSource.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
updateDashboard(data);
|
|
} catch (err) {
|
|
console.error('Error parsing SSE event:', err);
|
|
}
|
|
};
|
|
|
|
evtSource.onerror = (err) => {
|
|
console.warn('SSE disconnected, retrying...', err);
|
|
document.getElementById('sse-badge').textContent = 'RECONNECTING...';
|
|
document.getElementById('sse-badge').className = 'text-rose-400 font-bold';
|
|
};
|
|
}
|
|
|
|
function updateDashboard(data) {
|
|
if (!data) return;
|
|
|
|
// Governor and arbitration state ride along in the shared snapshot.
|
|
if (data.governor) renderGovernor(data.governor);
|
|
if (data.arbitrator) renderArbitrator(data.arbitrator);
|
|
|
|
// 1. GPU VRAM Stats
|
|
const gpu = data.gpu || {};
|
|
const ram = data.ram || {};
|
|
if (gpu.available) {
|
|
pushOverclockSample(gpu, ram);
|
|
document.getElementById('gpu-chip-name').textContent = gpu.device_name || 'NVIDIA GPU';
|
|
document.getElementById('vram-total-used').textContent = gpu.vram_used_gb || '0.0';
|
|
document.getElementById('vram-used-pct').textContent = `${gpu.vram_used_pct || 0}% USED`;
|
|
|
|
const bd = gpu.breakdown || {};
|
|
const totalBytes = gpu.vram_total_bytes || (16 * 1024**3);
|
|
|
|
const ollamaPct = ((bd.ollama_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const comfyPct = ((bd.comfyui_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const systemPct = ((bd.system_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const freePct = Math.max(0, 100 - (ollamaPct + comfyPct + systemPct));
|
|
|
|
document.getElementById('bar-ollama').style.width = `${ollamaPct}%`;
|
|
document.getElementById('bar-comfy').style.width = `${comfyPct}%`;
|
|
document.getElementById('bar-system').style.width = `${systemPct}%`;
|
|
document.getElementById('bar-free').style.width = `${freePct}%`;
|
|
|
|
document.getElementById('tooltip-ollama').textContent = `${bd.ollama_gb || 0} GB`;
|
|
document.getElementById('tooltip-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
|
|
document.getElementById('tooltip-system').textContent = `${bd.system_gb || 0} GB`;
|
|
|
|
document.getElementById('legend-ollama').textContent = `${bd.ollama_gb || 0} GB`;
|
|
document.getElementById('legend-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
|
|
document.getElementById('legend-system').textContent = `${bd.system_gb || 0} GB`;
|
|
document.getElementById('legend-free').textContent = `${bd.free_gb || 0} GB`;
|
|
|
|
// Hardware sensors
|
|
document.getElementById('gpu-util-val').textContent = `${gpu.gpu_util_pct || 0}%`;
|
|
document.getElementById('gpu-temp-val').textContent = `${gpu.temperature_c || 0}°C`;
|
|
document.getElementById('gpu-power-val').textContent = `${gpu.power_w || 0} W`;
|
|
document.getElementById('gpu-fan-val').textContent = `${gpu.fan_pct || 0}%`;
|
|
|
|
// Per-fan and animations
|
|
const fan0 = (gpu.fans && gpu.fans.length > 0) ? gpu.fans[0] : (gpu.fan_pct || 0);
|
|
const fan1 = (gpu.fans && gpu.fans.length > 1) ? gpu.fans[1] : (gpu.fan_pct || 0);
|
|
const fanSub = document.getElementById('gpu-fan-sub');
|
|
if (fanSub) fanSub.textContent = `Fan 0: ${fan0}% | Fan 1: ${fan1}%`;
|
|
|
|
const fan0Val = document.getElementById('oc-fan0-val');
|
|
if (fan0Val) fan0Val.textContent = `${fan0}%`;
|
|
const fan0Bar = document.getElementById('oc-fan0-bar');
|
|
if (fan0Bar) fan0Bar.style.width = `${fan0}%`;
|
|
|
|
const fan1Val = document.getElementById('oc-fan1-val');
|
|
if (fan1Val) fan1Val.textContent = `${fan1}%`;
|
|
const fan1Bar = document.getElementById('oc-fan1-bar');
|
|
if (fan1Bar) fan1Bar.style.width = `${fan1}%`;
|
|
|
|
const fanCurrent = document.getElementById('oc-fan-current');
|
|
if (fanCurrent) fanCurrent.textContent = `${gpu.fan_pct || 0}%`;
|
|
|
|
const spinSpeed = Math.max(0.2, (100 - (gpu.fan_pct || 0)) / 100 * 1.6 + 0.3);
|
|
const fanIcon = document.getElementById('gpu-fan-icon');
|
|
if (fanIcon) {
|
|
if ((gpu.fan_pct || 0) > 0) {
|
|
fanIcon.classList.add('fan-spinning');
|
|
fanIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
|
|
} else {
|
|
fanIcon.classList.remove('fan-spinning');
|
|
}
|
|
}
|
|
const ocFanCardIcon = document.getElementById('oc-fan-card-icon');
|
|
if (ocFanCardIcon) {
|
|
if ((gpu.fan_pct || 0) > 0) {
|
|
ocFanCardIcon.classList.add('fan-spinning');
|
|
ocFanCardIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
|
|
} else {
|
|
ocFanCardIcon.classList.remove('fan-spinning');
|
|
}
|
|
}
|
|
|
|
// Processes table
|
|
const tbody = document.getElementById('gpu-proc-table');
|
|
if (bd.processes && bd.processes.length > 0) {
|
|
tbody.innerHTML = bd.processes.map(p => {
|
|
let tagClass = 'text-slate-400';
|
|
let badge = '';
|
|
if (p.is_ollama) {
|
|
tagClass = 'text-purple-400 font-bold';
|
|
badge = '<span class="px-1 py-0.2 bg-purple-950 text-purple-300 rounded border border-purple-800 text-[9px] mr-1">OLLAMA</span>';
|
|
} else if (p.is_comfy) {
|
|
tagClass = 'text-cyan-400 font-bold';
|
|
badge = '<span class="px-1 py-0.2 bg-cyan-950 text-cyan-300 rounded border border-cyan-800 text-[9px] mr-1">COMFY</span>';
|
|
}
|
|
return `
|
|
<tr class="hover:bg-slate-900/50">
|
|
<td class="py-1 text-slate-500 font-mono text-[10px]">${p.pid}</td>
|
|
<td class="py-1 ${tagClass} text-[11px] truncate max-w-[140px]">${badge}${p.name}</td>
|
|
<td class="py-1 text-right font-mono font-bold text-slate-200 text-[11px]">${p.vram_mb} MB</td>
|
|
</tr>
|
|
`;
|
|
}).join('');
|
|
} else {
|
|
tbody.innerHTML = '<tr><td colspan="3" class="py-2 text-center text-slate-500">No compute processes running</td></tr>';
|
|
}
|
|
}
|
|
|
|
// 2. System RAM & Page Cache
|
|
if (ram.total_bytes) {
|
|
document.getElementById('ram-cached-gb').textContent = ram.cached_gb || '0.0';
|
|
document.getElementById('ram-total-text').textContent = `${ram.total_gb || 0} GB Total (${ram.cache_ratio_pct || 0}% in Cache)`;
|
|
|
|
const usedPct = ((ram.used_bytes || 0) / ram.total_bytes) * 100;
|
|
const cachePct = ((ram.cached_bytes || 0) / ram.total_bytes) * 100;
|
|
const freePct = Math.max(0, 100 - (usedPct + cachePct));
|
|
|
|
document.getElementById('bar-ram-used').style.width = `${usedPct}%`;
|
|
document.getElementById('bar-ram-cache').style.width = `${cachePct}%`;
|
|
document.getElementById('bar-ram-free').style.width = `${freePct}%`;
|
|
|
|
document.getElementById('legend-ram-used').textContent = `${ram.used_gb || 0} GB`;
|
|
document.getElementById('legend-ram-cached').textContent = `${ram.cached_gb || 0} GB`;
|
|
document.getElementById('legend-ram-free').textContent = `${ram.free_gb || 0} GB`;
|
|
}
|
|
|
|
// 3. Ollama State
|
|
const ollama = data.ollama || {};
|
|
if (ollama.online) {
|
|
document.getElementById('ollama-status-text').textContent = 'ONLINE';
|
|
document.getElementById('ollama-status-text').className = 'text-emerald-400 font-mono font-bold';
|
|
document.getElementById('ollama-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-emerald-500';
|
|
document.getElementById('ollama-pulse').className = 'animate-ping absolute inline-flex h-full w-full rounded-full bg-emerald-400 opacity-75';
|
|
|
|
if (ollama.active_model_name) {
|
|
document.getElementById('ollama-active-model').textContent = ollama.active_model_name;
|
|
document.getElementById('ollama-vram-badge').textContent = `${ollama.active_model_vram_gb || 0} GB VRAM`;
|
|
document.getElementById('ollama-vram-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-purple-900/80 text-purple-200 border border-purple-600 font-bold';
|
|
document.getElementById('ollama-context').textContent = `${ollama.active_context || 0} ctx`;
|
|
} else {
|
|
document.getElementById('ollama-active-model').textContent = 'None Loaded (VRAM Free)';
|
|
document.getElementById('ollama-vram-badge').textContent = '0.0 GB VRAM';
|
|
document.getElementById('ollama-vram-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-slate-800 text-slate-400 border border-slate-700';
|
|
document.getElementById('ollama-context').textContent = 'Idle';
|
|
}
|
|
|
|
if (ollama.installed_models && ollama.installed_models.length > 0) {
|
|
document.getElementById('ollama-total-models').textContent = ollama.installed_models.length;
|
|
updateModelSelect(ollama.installed_models, ollama.active_model_name);
|
|
}
|
|
} else {
|
|
document.getElementById('ollama-status-text').textContent = 'OFFLINE';
|
|
document.getElementById('ollama-status-text').className = 'text-rose-400 font-mono font-bold';
|
|
document.getElementById('ollama-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-rose-500';
|
|
document.getElementById('ollama-pulse').className = 'hidden';
|
|
}
|
|
|
|
// 4. ComfyUI State
|
|
const comfy = data.comfyui || {};
|
|
if (comfy.online) {
|
|
document.getElementById('comfy-status-text').textContent = 'ONLINE';
|
|
document.getElementById('comfy-status-text').className = 'text-emerald-400 font-mono font-bold';
|
|
document.getElementById('comfy-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-emerald-500';
|
|
document.getElementById('comfy-pulse').className = 'animate-ping absolute inline-flex h-full w-full rounded-full bg-emerald-400 opacity-75';
|
|
|
|
document.getElementById('comfy-queue-count').textContent = comfy.queue_running + comfy.queue_remaining;
|
|
if (comfy.executing) {
|
|
document.getElementById('comfy-exec-badge').textContent = 'GENERATING...';
|
|
document.getElementById('comfy-exec-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-amber-950 text-amber-300 border border-amber-600 animate-pulse font-bold';
|
|
document.getElementById('comfy-model-info').textContent = `Prompt #${comfy.current_prompt_id || 'Active'}`;
|
|
} else {
|
|
document.getElementById('comfy-exec-badge').textContent = 'IDLE / READY';
|
|
document.getElementById('comfy-exec-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-emerald-950 text-emerald-300 border border-emerald-800';
|
|
document.getElementById('comfy-model-info').textContent = 'Dynamic Model Offloader';
|
|
}
|
|
|
|
document.getElementById('comfy-vram-avail').textContent = `${(comfy.vram_free_mb / 1024).toFixed(1)} GB`;
|
|
} else {
|
|
document.getElementById('comfy-status-text').textContent = 'OFFLINE';
|
|
document.getElementById('comfy-status-text').className = 'text-rose-400 font-mono font-bold';
|
|
document.getElementById('comfy-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-rose-500';
|
|
document.getElementById('comfy-pulse').className = 'hidden';
|
|
}
|
|
|
|
document.getElementById('comfy-discovered-count').textContent = `${data.comfy_models_count || 0} Files`;
|
|
|
|
// 5. Switch History Timeline
|
|
const history = data.history || [];
|
|
const logContainer = document.getElementById('switch-log-container');
|
|
if (history.length > 0) {
|
|
const latest = history[0];
|
|
document.getElementById('ollama-last-swap').textContent = `${latest.duration_ms} ms`;
|
|
document.getElementById('ollama-cache-hit').textContent = latest.cache_status || 'OK';
|
|
|
|
logContainer.innerHTML = history.slice(0, 10).map(item => {
|
|
const isHit = (item.cache_status || '').includes('RAM Cache Hit') || (item.cache_status || '').includes('RAM-Cached');
|
|
const badgeClass = isHit
|
|
? 'bg-emerald-950/80 text-emerald-300 border-emerald-800'
|
|
: 'bg-amber-950/80 text-amber-300 border-amber-800';
|
|
return `
|
|
<div class="p-2.5 rounded-xl bg-slate-950 border border-slate-800 text-xs font-mono flex items-center justify-between">
|
|
<div class="space-y-0.5">
|
|
<div class="flex items-center space-x-1.5">
|
|
<span class="text-slate-500 text-[10px]">${item.timestamp}</span>
|
|
<span class="text-indigo-400 font-bold">${item.event_type}</span>
|
|
</div>
|
|
<div class="text-[11px] text-slate-300 truncate max-w-[260px]">
|
|
<span class="text-slate-500">${item.source}</span> → <span class="text-cyan-300 font-bold">${item.target}</span>
|
|
</div>
|
|
</div>
|
|
<div class="text-right">
|
|
<div class="text-slate-100 font-bold">${item.duration_ms} ms</div>
|
|
<span class="px-1.5 py-0.2 text-[9px] rounded border ${badgeClass}">${item.cache_status}</span>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
}
|
|
}
|
|
|
|
function updateModelSelect(models, activeModel) {
|
|
const select = document.getElementById('ollama-model-select');
|
|
const currentVal = select.value;
|
|
|
|
if (JSON.stringify(models.map(m => m.name)) === JSON.stringify(currentInstalledModels)) {
|
|
return;
|
|
}
|
|
|
|
currentInstalledModels = models.map(m => m.name);
|
|
select.innerHTML = models.map(m => {
|
|
const isSelected = m.name === activeModel || m.name === currentVal;
|
|
const sizeGb = (m.size / (1024**3)).toFixed(1);
|
|
const quant = m.details?.quantization_level || '';
|
|
return `<option value="${m.name}" ${isSelected ? 'selected' : ''}>${m.name} (${sizeGb} GB ${quant})</option>`;
|
|
}).join('');
|
|
}
|
|
|
|
// User Actions
|
|
async function triggerModelSwitch() {
|
|
const select = document.getElementById('ollama-model-select');
|
|
const targetModel = select.value;
|
|
if (!targetModel) return;
|
|
|
|
const btn = document.getElementById('btn-switch-model');
|
|
btn.disabled = true;
|
|
btn.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i> <span>Swapping...</span>';
|
|
|
|
try {
|
|
const resp = await fetch('/api/switch-model', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ model: targetModel, keep_alive: '30m' })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) {
|
|
alert(`Switch failed: ${result.detail || 'Unknown error'}`);
|
|
}
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
} finally {
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="fa-solid fa-shuffle"></i> <span>Hot Swap</span>';
|
|
}
|
|
}
|
|
|
|
async function freeOllamaVRAM() {
|
|
try {
|
|
const resp = await fetch('/api/free-vram', { method: 'POST' });
|
|
const data = await resp.json();
|
|
console.log('Ollama VRAM yielded:', data);
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function freeComfyVRAM() {
|
|
try {
|
|
const resp = await fetch('/api/comfy-free', { method: 'POST' });
|
|
const data = await resp.json();
|
|
console.log('ComfyUI VRAM freed:', data);
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function warmAllModels() {
|
|
try {
|
|
const resp = await fetch('/api/warm-all', { method: 'POST' });
|
|
const data = await resp.json();
|
|
alert(`Warming completed in ${data.total_duration_ms} ms! All models are now cached in 64GB RAM.`);
|
|
} catch (err) {
|
|
alert(`Error warming models: ${err}`);
|
|
}
|
|
}
|
|
|
|
// Startup
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
initSSE();
|
|
initOverclockChart();
|
|
fetchOverclockStatus();
|
|
setInterval(fetchOverclockStatus, 3000);
|
|
});
|
|
|
|
// ============ OVERCLOCK CONTROL ============
|
|
let ocProfiles = {};
|
|
let ocChart = null;
|
|
const OC_MAX_SAMPLES = 120;
|
|
|
|
function initOverclockChart() {
|
|
const canvas = document.getElementById('oc-chart');
|
|
if (!canvas || typeof Chart === 'undefined') return;
|
|
const ctx = canvas.getContext('2d');
|
|
|
|
ocChart = new Chart(ctx, {
|
|
type: 'line',
|
|
data: {
|
|
labels: [],
|
|
datasets: [
|
|
{ label: 'Core MHz', data: [], borderColor: '#22d3ee', backgroundColor: 'rgba(34,211,238,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: true },
|
|
{ label: 'Mem MHz', data: [], borderColor: '#c084fc', backgroundColor: 'rgba(192,132,252,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: false },
|
|
{ label: 'Temp °C', data: [], borderColor: '#fb7185', backgroundColor: 'rgba(251,113,133,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y1', fill: false },
|
|
{ label: 'Power W', data: [], borderColor: '#fbbf24', backgroundColor: 'rgba(251,191,36,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y2', fill: false },
|
|
{ label: 'Fan %', data: [], borderColor: '#34d399', backgroundColor: 'rgba(52,211,153,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y3', fill: false },
|
|
{ label: 'VRAM GB', data: [], borderColor: '#818cf8', backgroundColor: 'rgba(129,140,248,0.08)', borderWidth: 1.8, pointRadius: 0, tension: 0.35, yAxisID: 'y4', fill: false },
|
|
{ label: 'RAM Cache GB', data: [], borderColor: '#e879f9', backgroundColor: 'rgba(232,121,249,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y5', fill: false },
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
animation: false,
|
|
interaction: { mode: 'index', intersect: false },
|
|
scales: {
|
|
x: { ticks: { color: '#64748b', maxTicksLimit: 8, font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
|
|
y: { position: 'left', title: { display: true, text: 'MHz', color: '#22d3ee', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
|
|
y1: { position: 'right', title: { display: true, text: '°C', color: '#fb7185', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
|
|
y2: { position: 'right', offset: true, title: { display: true, text: 'W', color: '#fbbf24', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 400 },
|
|
y3: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
|
|
y4: { position: 'right', offset: true, title: { display: true, text: 'VRAM GB', color: '#818cf8', font: { size: 9 } }, ticks: { color: '#818cf8', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 16 },
|
|
y5: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 64 },
|
|
},
|
|
plugins: {
|
|
legend: { display: false },
|
|
tooltip: {
|
|
callbacks: {
|
|
label: function(context) {
|
|
const label = context.dataset.label || '';
|
|
const val = context.parsed.y;
|
|
if (label.includes('MHz')) return `${label}: ${val} MHz`;
|
|
if (label.includes('°C')) return `${label}: ${val}°C`;
|
|
if (label.includes('Power')) return `${label}: ${val} W`;
|
|
if (label.includes('Fan')) return `${label}: ${val}%`;
|
|
if (label.includes('GB')) return `${label}: ${val} GB`;
|
|
return `${label}: ${val}`;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
}
|
|
});
|
|
}
|
|
|
|
function pushOverclockSample(gpu, ram) {
|
|
if (!ocChart || !gpu || !gpu.available) return;
|
|
const label = new Date().toLocaleTimeString([], { hour12: false });
|
|
ocChart.data.labels.push(label);
|
|
ocChart.data.datasets[0].data.push(gpu.clock_graphics_mhz || 0);
|
|
ocChart.data.datasets[1].data.push(gpu.clock_mem_mhz || 0);
|
|
ocChart.data.datasets[2].data.push(gpu.temperature_c || 0);
|
|
ocChart.data.datasets[3].data.push(gpu.power_w || 0);
|
|
ocChart.data.datasets[4].data.push(gpu.fan_pct || 0);
|
|
ocChart.data.datasets[5].data.push(parseFloat(gpu.vram_used_gb) || 0);
|
|
ocChart.data.datasets[6].data.push(parseFloat(ram?.cached_gb) || 0);
|
|
if (ocChart.data.labels.length > OC_MAX_SAMPLES) {
|
|
ocChart.data.labels.shift();
|
|
ocChart.data.datasets.forEach(d => d.data.shift());
|
|
}
|
|
ocChart.update('none');
|
|
}
|
|
|
|
let initialOcLoaded = false;
|
|
|
|
async function fetchOverclockStatus() {
|
|
try {
|
|
const resp = await fetch('/api/overclock');
|
|
if (!resp.ok) return;
|
|
const data = await resp.json();
|
|
ocProfiles = data.profiles || {};
|
|
renderOverclockStatus(data);
|
|
if (!initialOcLoaded && data.active_profile) {
|
|
initialOcLoaded = true;
|
|
const editSel = document.getElementById('oc-edit-profile');
|
|
if (editSel) {
|
|
editSel.value = data.active_profile;
|
|
loadOverclockForProfile(data.active_profile);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.warn('Overclock fetch error:', err);
|
|
}
|
|
}
|
|
|
|
function renderOverclockStatus(data) {
|
|
const active = data.active_profile || 'balanced';
|
|
|
|
const badge = document.getElementById('oc-active-badge');
|
|
badge.textContent = `Active: ${active}`;
|
|
if (active === 'ollama') {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-purple-950 border border-purple-700 text-purple-300';
|
|
} else if (active === 'comfy') {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-cyan-950 border border-cyan-700 text-cyan-300';
|
|
} else {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-300';
|
|
}
|
|
|
|
const gpu = data.gpu || {};
|
|
document.getElementById('oc-power-limit').textContent = `${gpu.power_limit_w ?? '--'} W`;
|
|
document.getElementById('oc-core').textContent = `${gpu.clock_sm_mhz ?? '--'} MHz`;
|
|
document.getElementById('oc-mem').textContent = `${gpu.clock_mem_mhz ?? '--'} MHz`;
|
|
document.getElementById('oc-temp-draw').textContent = `${gpu.temp_c ?? '--'}°C / ${gpu.power_draw_w ?? '--'}W`;
|
|
|
|
// Specs strip
|
|
document.getElementById('oc-spec-gpu').textContent = gpu.name || '--';
|
|
document.getElementById('oc-spec-driver').textContent = gpu.driver_version || '--';
|
|
const vramGb = (gpu.vram_total_mb || 0) / 1024;
|
|
document.getElementById('oc-spec-vram').textContent = vramGb > 0 ? vramGb.toFixed(0) + ' GB' : '--';
|
|
document.getElementById('oc-spec-maxcore').textContent = (gpu.clock_sm_max_mhz ?? '--') + ' MHz';
|
|
document.getElementById('oc-spec-maxmem').textContent = (gpu.clock_mem_max_mhz ?? '--') + ' MHz';
|
|
document.getElementById('oc-spec-power').textContent = `${gpu.power_limit_w ?? '--'} / ${gpu.power_max_w ?? '--'} W`;
|
|
|
|
// Fan status
|
|
const fan = data.fan || {};
|
|
const fanBadge = document.getElementById('oc-fan-badge');
|
|
if (fanBadge) {
|
|
if (fan.manual) {
|
|
fanBadge.textContent = `MANUAL (${fan.target_speed_pct ?? '--'}%)`;
|
|
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-500 text-emerald-300 font-bold';
|
|
} else {
|
|
fanBadge.textContent = 'AUTO (VBIOS)';
|
|
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-400 font-bold';
|
|
}
|
|
}
|
|
|
|
const xBadge = document.getElementById('oc-x-badge');
|
|
if (data.headless_x_running) {
|
|
xBadge.textContent = 'X: ON';
|
|
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-700 text-emerald-300';
|
|
} else {
|
|
xBadge.textContent = 'X: OFF';
|
|
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-rose-950 border border-rose-700 text-rose-300';
|
|
}
|
|
|
|
const offsets = (data.last_result && data.last_result.offsets) || {};
|
|
const note = document.getElementById('oc-offset-note');
|
|
if (offsets.supported) {
|
|
note.textContent = '✅ Clock offsets active (proprietary driver)';
|
|
note.className = 'text-[11px] font-mono text-emerald-400';
|
|
} else {
|
|
note.textContent = '⚠️ Clock offsets off — nvidia-open lacks offset support (power + clock locks active)';
|
|
note.className = 'text-[11px] font-mono text-amber-400';
|
|
}
|
|
|
|
// highlight active profile button
|
|
['ollama', 'comfy', 'balanced'].forEach(p => {
|
|
const btn = document.getElementById('oc-btn-' + p);
|
|
if (p === active) {
|
|
btn.classList.add('ring-2', 'ring-fuchsia-400');
|
|
} else {
|
|
btn.classList.remove('ring-2', 'ring-fuchsia-400');
|
|
}
|
|
});
|
|
}
|
|
|
|
function loadOverclockForProfile(name) {
|
|
const p = ocProfiles[name];
|
|
if (!p) return;
|
|
document.getElementById('oc-slider-power').value = p.power_limit_w || 370;
|
|
document.getElementById('oc-val-power').textContent = (p.power_limit_w || 370) + ' W';
|
|
document.getElementById('oc-slider-core').value = p.core_offset_mhz || 0;
|
|
document.getElementById('oc-val-core').textContent = '+' + (p.core_offset_mhz || 0) + ' MHz';
|
|
document.getElementById('oc-slider-mem').value = p.mem_offset_mhz || 0;
|
|
document.getElementById('oc-val-mem').textContent = '+' + (p.mem_offset_mhz || 0) + ' MHz';
|
|
document.getElementById('oc-lockcore-toggle').checked = (p.lock_core_max || 0) > 0;
|
|
document.getElementById('oc-val-lockcore').textContent = (p.lock_core_max || 0) > 0 ? 'On' : 'Off';
|
|
document.getElementById('oc-lockmem-toggle').checked = (p.lock_mem_mhz || 0) > 0;
|
|
document.getElementById('oc-val-lockmem').textContent = (p.lock_mem_mhz || 0) > 0 ? 'On' : 'Off';
|
|
|
|
const fanMode = p.fan_mode || 'auto';
|
|
const profFanMode = document.getElementById('oc-prof-fanmode');
|
|
if (profFanMode) profFanMode.value = fanMode;
|
|
const profFanSpeed = document.getElementById('oc-slider-prof-fanspeed');
|
|
if (profFanSpeed) profFanSpeed.value = p.fan_speed_pct || 70;
|
|
const profFanVal = document.getElementById('oc-val-prof-fanspeed');
|
|
if (profFanVal) profFanVal.textContent = (p.fan_speed_pct || 70) + '%';
|
|
toggleProfileFanMode();
|
|
}
|
|
|
|
function toggleProfileFanMode() {
|
|
const mode = document.getElementById('oc-prof-fanmode').value;
|
|
const container = document.getElementById('oc-prof-fanspeed-container');
|
|
const valBadge = document.getElementById('oc-val-prof-fanmode');
|
|
if (valBadge) valBadge.textContent = (mode === 'manual') ? 'Manual Target' : 'Auto (VBIOS)';
|
|
if (container) {
|
|
if (mode === 'manual') {
|
|
container.classList.remove('opacity-50', 'pointer-events-none');
|
|
} else {
|
|
container.classList.add('opacity-50', 'pointer-events-none');
|
|
}
|
|
}
|
|
}
|
|
|
|
function toggleCoreLock() {
|
|
document.getElementById('oc-val-lockcore').textContent = document.getElementById('oc-lockcore-toggle').checked ? 'On' : 'Off';
|
|
}
|
|
|
|
function toggleMemLock() {
|
|
document.getElementById('oc-val-lockmem').textContent = document.getElementById('oc-lockmem-toggle').checked ? 'On' : 'Off';
|
|
}
|
|
|
|
async function setFanManual(speed) {
|
|
let pct = speed;
|
|
if (pct === undefined || pct === null) {
|
|
pct = parseInt(document.getElementById('oc-fan-slider').value);
|
|
} else {
|
|
pct = parseInt(pct);
|
|
const slider = document.getElementById('oc-fan-slider');
|
|
if (slider) slider.value = pct;
|
|
const label = document.getElementById('oc-fan-slider-label');
|
|
if (label) label.textContent = pct + '%';
|
|
}
|
|
try {
|
|
const resp = await fetch('/api/overclock/fan', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ mode: 'manual', percent: pct })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) alert(`Fan set failed: ${result.detail || 'error'}`);
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error setting fan speed: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function setFanAuto() {
|
|
try {
|
|
const resp = await fetch('/api/overclock/fan', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ mode: 'auto' })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) alert(`Fan auto failed: ${result.detail || 'error'}`);
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error setting fan to auto: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function applyOverclock(profile) {
|
|
try {
|
|
const resp = await fetch('/api/overclock/apply', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ profile })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) {
|
|
alert(`Apply failed: ${result.detail || 'error'}`);
|
|
}
|
|
const editSel = document.getElementById('oc-edit-profile');
|
|
if (editSel) {
|
|
editSel.value = profile;
|
|
loadOverclockForProfile(profile);
|
|
}
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function saveOverclockProfile() {
|
|
const name = document.getElementById('oc-edit-profile').value;
|
|
const power = parseInt(document.getElementById('oc-slider-power').value);
|
|
const core = parseInt(document.getElementById('oc-slider-core').value);
|
|
const mem = parseInt(document.getElementById('oc-slider-mem').value);
|
|
const lockCore = document.getElementById('oc-lockcore-toggle').checked;
|
|
const lockMem = document.getElementById('oc-lockmem-toggle').checked;
|
|
const fanMode = document.getElementById('oc-prof-fanmode') ? document.getElementById('oc-prof-fanmode').value : 'auto';
|
|
const fanSpeed = document.getElementById('oc-slider-prof-fanspeed') ? parseInt(document.getElementById('oc-slider-prof-fanspeed').value) : 70;
|
|
|
|
const config = {
|
|
power_limit_w: power,
|
|
core_offset_mhz: core,
|
|
mem_offset_mhz: mem,
|
|
lock_core_min: lockCore ? 2900 : 0,
|
|
lock_core_max: lockCore ? 3105 : 0,
|
|
lock_mem_mhz: lockMem ? 11501 : 0,
|
|
fan_mode: fanMode,
|
|
fan_speed_pct: (fanMode === 'manual') ? fanSpeed : 0,
|
|
};
|
|
|
|
try {
|
|
const resp = await fetch(`/api/overclock/profiles/${name}`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ config })
|
|
});
|
|
const result = await resp.json();
|
|
if (resp.ok) {
|
|
await fetchOverclockStatus();
|
|
await applyOverclock(name);
|
|
} else {
|
|
alert(`Save failed: ${result.detail || 'error'}`);
|
|
}
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// THERMAL GOVERNOR / RESIDENCY / ANALYTICS / AUTOTUNE
|
|
// ============================================================================
|
|
|
|
let governorEnabled = true;
|
|
|
|
function renderGovernor(gov) {
|
|
if (!gov) return;
|
|
governorEnabled = gov.enabled;
|
|
const levels = 3;
|
|
const el = (id) => document.getElementById(id);
|
|
if (!el('gov-label')) return;
|
|
|
|
el('gov-label').textContent = gov.label || '—';
|
|
el('gov-label').className = 'text-2xl font-bold ' +
|
|
(gov.level === 0 ? 'text-emerald-400' : gov.level < 3 ? 'text-amber-400' : 'text-rose-400');
|
|
el('gov-level').textContent = `level ${gov.level} / ${levels}`;
|
|
el('gov-bar').style.width = `${(gov.level / levels) * 100}%`;
|
|
el('gov-esc').textContent = `${gov.escalate_at_c}°C`;
|
|
el('gov-rec').textContent = `${gov.recover_below_c}°C`;
|
|
el('gov-scale').textContent = `${Math.round((gov.offset_scale ?? 1) * 100)}%`;
|
|
el('gov-reason').textContent = gov.last_reason || '—';
|
|
|
|
const toggle = el('gov-toggle');
|
|
toggle.textContent = gov.enabled ? 'Enabled' : 'Disabled';
|
|
toggle.className = 'px-2.5 py-1 text-xs font-semibold rounded-lg border transition ' +
|
|
(gov.enabled ? 'bg-emerald-950/70 border-emerald-800 text-emerald-300 hover:bg-emerald-900'
|
|
: 'bg-slate-800 border-slate-700 text-slate-400 hover:bg-slate-700');
|
|
|
|
const hist = el('gov-history');
|
|
hist.innerHTML = (gov.history || []).map(h => {
|
|
const t = new Date(h.ts * 1000).toLocaleTimeString();
|
|
const up = h.to_level > h.from_level;
|
|
return `<div class="flex justify-between ${up ? 'text-rose-400' : 'text-cyan-400'}">
|
|
<span>${t} ${up ? '▲' : '▼'} L${h.to_level} ${h.label}</span>
|
|
<span class="text-slate-500 truncate ml-2">${h.reason || ''}</span></div>`;
|
|
}).join('') || '<div class="text-slate-600">No derates recorded — the card has stayed happy.</div>';
|
|
}
|
|
|
|
async function toggleGovernor() {
|
|
const res = await fetch('/api/governor', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ enabled: !governorEnabled })
|
|
});
|
|
renderGovernor(await res.json());
|
|
}
|
|
|
|
async function resetGovernor() {
|
|
const res = await fetch('/api/governor', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ reset: true })
|
|
});
|
|
renderGovernor(await res.json());
|
|
}
|
|
|
|
// ---------------------------------------------------------------- residency
|
|
|
|
async function fetchCacheReport(refresh = false) {
|
|
const body = document.getElementById('cache-files');
|
|
if (!body) return;
|
|
if (refresh) body.innerHTML = '<div class="text-slate-500 text-xs">Scanning…</div>';
|
|
try {
|
|
const res = await fetch(`/api/cache/report?refresh=${refresh ? 'true' : 'false'}`);
|
|
const d = await res.json();
|
|
|
|
document.getElementById('cache-total').textContent =
|
|
`${d.resident_total_gb} / ${d.catalog_total_gb} GB`;
|
|
document.getElementById('cache-meta').textContent =
|
|
`${d.residency_pct}% resident · ${d.warm_files}/${d.files_scanned} warm · ${d.scan_ms}ms`;
|
|
|
|
const cap = d.capability || {};
|
|
document.getElementById('cache-capability').innerHTML = cap.exact_everywhere
|
|
? '<span class="text-emerald-500">Exact measurement available for every model file.</span>'
|
|
: `<span class="text-amber-500">Ollama weights measured by read-rate probe</span> — ${cap.reason || ''}`;
|
|
|
|
body.innerHTML = (d.files || []).slice(0, 40).map(f => {
|
|
const pct = f.resident_pct ?? 0;
|
|
const color = pct >= 90 ? 'bg-emerald-500' : pct >= 25 ? 'bg-amber-500' : 'bg-slate-700';
|
|
const kind = f.kind === 'ollama' ? 'text-cyan-400' : 'text-fuchsia-400';
|
|
return `<div>
|
|
<div class="flex justify-between text-[11px] font-mono">
|
|
<span class="${kind} truncate mr-2" title="${f.full_path}">${f.name}</span>
|
|
<span class="text-slate-400 whitespace-nowrap">${pct}% · ${f.size_gb}GB</span>
|
|
</div>
|
|
<div class="w-full bg-slate-950 rounded-full h-1 mt-0.5 overflow-hidden">
|
|
<div class="${color} h-full transition-all" style="width:${pct}%"></div>
|
|
</div>
|
|
</div>`;
|
|
}).join('');
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400 text-xs">Residency scan failed: ${e}</div>`;
|
|
}
|
|
}
|
|
|
|
async function fetchWarmPlan() {
|
|
const body = document.getElementById('cache-files');
|
|
body.innerHTML = '<div class="text-slate-500 text-xs">Building plan…</div>';
|
|
try {
|
|
const d = await (await fetch('/api/warm-plan')).json();
|
|
document.getElementById('cache-total').textContent = `${d.planned_gb} GB planned`;
|
|
document.getElementById('cache-meta').textContent =
|
|
`budget ${d.budget_gb} GB · ${d.warm_count} to warm · ${d.skipped_count} skipped`;
|
|
document.getElementById('cache-capability').innerHTML =
|
|
'<span class="text-slate-500">Ranked by recency/frequency; already-resident files cost nothing.</span>';
|
|
body.innerHTML = d.plan.map((p, i) => `<div class="flex justify-between text-[11px] font-mono">
|
|
<span class="text-amber-400 truncate mr-2">${i + 1}. ${p.name}</span>
|
|
<span class="text-slate-400 whitespace-nowrap">${p.size_gb}GB · ${p.resident_pct}% · s=${p.score}</span>
|
|
</div>`).join('') || '<div class="text-emerald-500 text-xs">Everything worth warming is already resident.</div>';
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400 text-xs">${e}</div>`;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- analytics
|
|
|
|
async function fetchAnalytics() {
|
|
const el = document.getElementById('analytics-body');
|
|
if (!el) return;
|
|
const days = document.getElementById('analytics-days').value;
|
|
try {
|
|
const d = await (await fetch(`/api/analytics/profiles?days=${days}`)).json();
|
|
if (!d.profiles || !d.profiles.length) {
|
|
el.innerHTML = '<div class="text-slate-500">No swaps recorded yet in this window. Switch a model to start building the comparison.</div>';
|
|
return;
|
|
}
|
|
const best = Math.max(...d.profiles.map(p => p.avg_tok_s || 0));
|
|
el.innerHTML = `<table class="w-full text-[11px] font-mono">
|
|
<thead><tr class="text-slate-500 text-left border-b border-slate-800">
|
|
<th class="pb-1">profile</th><th>tok/s</th><th>load GB/s</th><th>°C avg</th><th>W avg</th><th>SM MHz</th><th>n</th>
|
|
</tr></thead><tbody>` +
|
|
d.profiles.map(p => {
|
|
const win = (p.avg_tok_s || 0) === best;
|
|
return `<tr class="border-b border-slate-900 ${win ? 'text-emerald-400' : 'text-slate-300'}">
|
|
<td class="py-1">${win ? '★ ' : ''}${p.profile ?? '—'}</td>
|
|
<td>${p.avg_tok_s ?? '—'}</td><td>${p.avg_load_gbps ?? '—'}</td>
|
|
<td>${p.avg_temp_c ?? '—'}</td><td>${p.avg_power_w ?? '—'}</td>
|
|
<td>${p.avg_clock_sm ?? '—'}</td><td class="text-slate-500">${p.swaps}</td></tr>`;
|
|
}).join('') + '</tbody></table>';
|
|
} catch (e) {
|
|
el.innerHTML = `<div class="text-rose-400">${e}</div>`;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- autotune
|
|
|
|
let sweepPoll = null;
|
|
|
|
async function runSweep() {
|
|
const btn = document.getElementById('at-run');
|
|
const body = document.getElementById('autotune-body');
|
|
const payload = {
|
|
knob: document.getElementById('at-knob').value,
|
|
profile: document.getElementById('at-profile').value,
|
|
start: parseInt(document.getElementById('at-start').value, 10),
|
|
stop: parseInt(document.getElementById('at-stop').value, 10),
|
|
step: parseInt(document.getElementById('at-step').value, 10),
|
|
apply_best: document.getElementById('at-apply').checked,
|
|
};
|
|
btn.disabled = true;
|
|
btn.classList.add('opacity-50');
|
|
body.innerHTML = '<div class="text-fuchsia-400">Sweeping… each step loads the model and benchmarks decode. This takes a few minutes.</div>';
|
|
sweepPoll = setInterval(pollSweep, 2000);
|
|
try {
|
|
const res = await fetch('/api/autotune/sweep', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload)
|
|
});
|
|
const d = await res.json();
|
|
renderSweep(res.ok ? d : { error: d.detail });
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400">${e}</div>`;
|
|
} finally {
|
|
clearInterval(sweepPoll);
|
|
btn.disabled = false;
|
|
btn.classList.remove('opacity-50');
|
|
document.getElementById('autotune-state').textContent = 'idle';
|
|
}
|
|
}
|
|
|
|
async function pollSweep() {
|
|
try {
|
|
const d = await (await fetch('/api/autotune')).json();
|
|
const s = document.getElementById('autotune-state');
|
|
if (d.running && d.current) {
|
|
s.textContent = `${d.current.knob}=${d.current.value} → ${d.current.tokens_per_sec} tok/s`;
|
|
s.className = 'text-xs font-mono text-fuchsia-400';
|
|
}
|
|
} catch (e) { /* transient */ }
|
|
}
|
|
|
|
async function cancelSweep() {
|
|
await fetch('/api/autotune/cancel', { method: 'POST' });
|
|
document.getElementById('autotune-state').textContent = 'cancelling…';
|
|
}
|
|
|
|
function renderSweep(d) {
|
|
const body = document.getElementById('autotune-body');
|
|
if (d.error) { body.innerHTML = `<div class="text-rose-400">${d.error}</div>`; return; }
|
|
const rows = (d.table || []).map(r => `<tr class="${r.stable ? 'text-slate-300' : 'text-rose-400'} border-b border-slate-900">
|
|
<td class="py-0.5">${r.value}</td><td>${r.tokens_per_sec}</td><td>${r.temp_c}</td>
|
|
<td>${r.power_w ?? '—'}</td><td>${r.clock_mem_mhz ?? '—'}</td>
|
|
<td class="text-slate-500 truncate">${r.instability || 'stable'}</td></tr>`).join('');
|
|
const best = d.best
|
|
? `<div class="text-emerald-400 mb-2">★ Best stable: <span class="font-mono">${d.knob}=${d.best.value}</span>
|
|
→ ${d.best.tokens_per_sec} tok/s at ${d.best.temp_c}°C
|
|
${d.gain_vs_current_pct != null
|
|
? `<span class="text-slate-400">(${d.gain_vs_current_pct > 0 ? '+' : ''}${d.gain_vs_current_pct}% vs your current setting)</span>`
|
|
: d.gain_vs_first_step_pct != null
|
|
? `<span class="text-slate-500">(spread across tried values: ${d.gain_vs_first_step_pct > 0 ? '+' : ''}${d.gain_vs_first_step_pct}% vs slowest)</span>`
|
|
: ''}
|
|
${d.applied_to_profile ? '<span class="text-fuchsia-400">· saved to profile</span>' : ''}</div>`
|
|
: '<div class="text-amber-400 mb-2">No stable step produced throughput.</div>';
|
|
const unstable = d.first_unstable
|
|
? `<div class="text-rose-400 text-[11px] mb-2">Stopped climbing at ${d.first_unstable.value}: ${d.first_unstable.why}</div>` : '';
|
|
body.innerHTML = best + unstable + `<table class="w-full text-[11px] font-mono">
|
|
<thead><tr class="text-slate-500 text-left border-b border-slate-800">
|
|
<th class="pb-1">offset</th><th>tok/s</th><th>°C</th><th>W</th><th>mem MHz</th><th>status</th>
|
|
</tr></thead><tbody>${rows}</tbody></table>`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- wiring
|
|
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
fetchCacheReport(false);
|
|
fetchAnalytics();
|
|
setInterval(fetchAnalytics, 60000);
|
|
setInterval(() => fetchCacheReport(false), 30000);
|
|
fetch('/api/autotune').then(r => r.json()).then(d => {
|
|
if (d.last_result) renderSweep(d.last_result);
|
|
}).catch(() => {});
|
|
});
|
|
|
|
|
|
// ---------------------------------------------------------------- arbitration
|
|
|
|
function renderArbitrator(arb) {
|
|
const el = (id) => document.getElementById(id);
|
|
if (!el('arb-action')) return;
|
|
const c = arb.counters || {};
|
|
|
|
el('arb-action').textContent = arb.last_action || 'Idle';
|
|
el('arb-yields').textContent = c.yields ?? 0;
|
|
el('arb-busy').textContent = c.yield_deferred_busy ?? 0;
|
|
el('arb-later').textContent = c.deferred_releases ?? 0;
|
|
el('arb-stalled').textContent = c.yield_stalled ?? 0;
|
|
el('arb-purges').textContent = c.purges ?? 0;
|
|
el('arb-defpurge').textContent = c.deferred_purges ?? 0;
|
|
|
|
const ws = el('arb-ws');
|
|
ws.textContent = arb.connected_ws ? 'ComfyUI WS live' : 'WS down — polling';
|
|
ws.className = 'text-xs font-mono ' + (arb.connected_ws ? 'text-emerald-400' : 'text-amber-400');
|
|
|
|
// Show why we are holding off, and the idle countdown before ComfyUI is purged.
|
|
const parts = [];
|
|
const backoff = arb.yield_backoff || {};
|
|
for (const [model, secs] of Object.entries(backoff)) {
|
|
parts.push(`waiting ${secs}s before asking '${model}' again`);
|
|
}
|
|
if (arb.pending_purge && arb.comfy_idle_s != null) {
|
|
const left = Math.max((arb.idle_purge_after_s || 0) - arb.comfy_idle_s, 0).toFixed(0);
|
|
parts.push(`ComfyUI idle ${arb.comfy_idle_s}s — holding its checkpoints ${left}s longer`);
|
|
}
|
|
el('arb-backoff').textContent = parts.join(' · ');
|
|
}
|