let evtSource = null;
let currentInstalledModels = [];
// Initialize SSE Stream
function initSSE() {
if (evtSource) {
evtSource.close();
}
evtSource = new EventSource('/api/stream');
evtSource.onopen = () => {
document.getElementById('sse-badge').textContent = 'SSE LIVE';
document.getElementById('sse-badge').className = 'text-emerald-400 font-bold';
};
evtSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
updateDashboard(data);
} catch (err) {
console.error('Error parsing SSE event:', err);
}
};
evtSource.onerror = (err) => {
console.warn('SSE disconnected, retrying...', err);
document.getElementById('sse-badge').textContent = 'RECONNECTING...';
document.getElementById('sse-badge').className = 'text-rose-400 font-bold';
};
}
function updateDashboard(data) {
if (!data) return;
// Governor and arbitration state ride along in the shared snapshot.
if (data.governor) renderGovernor(data.governor);
if (data.arbitrator) renderArbitrator(data.arbitrator, data.gpu);
// 1. GPU VRAM Stats
const gpu = data.gpu || {};
const ram = data.ram || {};
if (gpu.available) {
pushOverclockSample(gpu, ram);
document.getElementById('gpu-chip-name').textContent = gpu.device_name || 'NVIDIA GPU';
document.getElementById('vram-total-used').textContent = gpu.vram_used_gb || '0.0';
document.getElementById('vram-used-pct').textContent = `${gpu.vram_used_pct || 0}% USED`;
const bd = gpu.breakdown || {};
const totalBytes = gpu.vram_total_bytes || (16 * 1024**3);
const ollamaPct = ((bd.ollama_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
const comfyPct = ((bd.comfyui_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
const systemPct = ((bd.system_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
const freePct = Math.max(0, 100 - (ollamaPct + comfyPct + systemPct));
document.getElementById('bar-ollama').style.width = `${ollamaPct}%`;
document.getElementById('bar-comfy').style.width = `${comfyPct}%`;
document.getElementById('bar-system').style.width = `${systemPct}%`;
document.getElementById('bar-free').style.width = `${freePct}%`;
document.getElementById('tooltip-ollama').textContent = `${bd.ollama_gb || 0} GB`;
document.getElementById('tooltip-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
document.getElementById('tooltip-system').textContent = `${bd.system_gb || 0} GB`;
document.getElementById('legend-ollama').textContent = `${bd.ollama_gb || 0} GB`;
document.getElementById('legend-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
document.getElementById('legend-system').textContent = `${bd.system_gb || 0} GB`;
document.getElementById('legend-free').textContent = `${bd.free_gb || 0} GB`;
// Hardware sensors
document.getElementById('gpu-util-val').textContent = `${gpu.gpu_util_pct || 0}%`;
document.getElementById('gpu-temp-val').textContent = `${gpu.temperature_c || 0}°C`;
document.getElementById('gpu-power-val').textContent = `${gpu.power_w || 0} W`;
document.getElementById('gpu-fan-val').textContent = `${gpu.fan_pct || 0}%`;
// Per-fan and animations
const fan0 = (gpu.fans && gpu.fans.length > 0) ? gpu.fans[0] : (gpu.fan_pct || 0);
const fan1 = (gpu.fans && gpu.fans.length > 1) ? gpu.fans[1] : (gpu.fan_pct || 0);
const fanSub = document.getElementById('gpu-fan-sub');
if (fanSub) fanSub.textContent = `Fan 0: ${fan0}% | Fan 1: ${fan1}%`;
const fan0Val = document.getElementById('oc-fan0-val');
if (fan0Val) fan0Val.textContent = `${fan0}%`;
const fan0Bar = document.getElementById('oc-fan0-bar');
if (fan0Bar) fan0Bar.style.width = `${fan0}%`;
const fan1Val = document.getElementById('oc-fan1-val');
if (fan1Val) fan1Val.textContent = `${fan1}%`;
const fan1Bar = document.getElementById('oc-fan1-bar');
if (fan1Bar) fan1Bar.style.width = `${fan1}%`;
const fanCurrent = document.getElementById('oc-fan-current');
if (fanCurrent) fanCurrent.textContent = `${gpu.fan_pct || 0}%`;
const spinSpeed = Math.max(0.2, (100 - (gpu.fan_pct || 0)) / 100 * 1.6 + 0.3);
const fanIcon = document.getElementById('gpu-fan-icon');
if (fanIcon) {
if ((gpu.fan_pct || 0) > 0) {
fanIcon.classList.add('fan-spinning');
fanIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
} else {
fanIcon.classList.remove('fan-spinning');
}
}
const ocFanCardIcon = document.getElementById('oc-fan-card-icon');
if (ocFanCardIcon) {
if ((gpu.fan_pct || 0) > 0) {
ocFanCardIcon.classList.add('fan-spinning');
ocFanCardIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
} else {
ocFanCardIcon.classList.remove('fan-spinning');
}
}
// Processes table
const tbody = document.getElementById('gpu-proc-table');
if (bd.processes && bd.processes.length > 0) {
tbody.innerHTML = bd.processes.map(p => {
let tagClass = 'text-slate-400';
let badge = '';
if (p.is_ollama) {
tagClass = 'text-purple-400 font-bold';
badge = 'OLLAMA ';
} else if (p.is_comfy) {
tagClass = 'text-cyan-400 font-bold';
badge = 'COMFY ';
}
return `
${p.pid}
${badge}${p.name}
${p.vram_mb} MB
`;
}).join('');
} else {
tbody.innerHTML = 'No compute processes running
${item.timestamp}
${item.event_type}
${item.source} → ${item.target}
${item.duration_ms} ms
${item.cache_status}
`;
}).join('');
}
}
function updateModelSelect(models, activeModel) {
const select = document.getElementById('ollama-model-select');
const currentVal = select.value;
if (JSON.stringify(models.map(m => m.name)) === JSON.stringify(currentInstalledModels)) {
return;
}
currentInstalledModels = models.map(m => m.name);
select.innerHTML = models.map(m => {
const isSelected = m.name === activeModel || m.name === currentVal;
const sizeGb = (m.size / (1024**3)).toFixed(1);
const quant = m.details?.quantization_level || '';
return `${m.name} (${sizeGb} GB ${quant}) `;
}).join('');
}
// User Actions
async function triggerModelSwitch() {
const select = document.getElementById('ollama-model-select');
const targetModel = select.value;
if (!targetModel) return;
const btn = document.getElementById('btn-switch-model');
btn.disabled = true;
btn.innerHTML = 'Swapping... ';
try {
const resp = await fetch('/api/switch-model', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model: targetModel, keep_alive: '30m' })
});
const result = await resp.json();
if (!resp.ok) {
alert(`Switch failed: ${result.detail || 'Unknown error'}`);
}
} catch (err) {
alert(`Error: ${err}`);
} finally {
btn.disabled = false;
btn.innerHTML = 'Hot Swap ';
}
}
async function freeOllamaVRAM() {
try {
const resp = await fetch('/api/free-vram', { method: 'POST' });
const data = await resp.json();
console.log('Ollama VRAM yielded:', data);
} catch (err) {
alert(`Error: ${err}`);
}
}
async function freeComfyVRAM() {
try {
const resp = await fetch('/api/comfy-free', { method: 'POST' });
const data = await resp.json();
console.log('ComfyUI VRAM freed:', data);
} catch (err) {
alert(`Error: ${err}`);
}
}
async function warmAllModels() {
try {
const resp = await fetch('/api/warm-all', { method: 'POST' });
const data = await resp.json();
alert(`Warming completed in ${data.total_duration_ms} ms! All models are now cached in 64GB RAM.`);
} catch (err) {
alert(`Error warming models: ${err}`);
}
}
// Startup
document.addEventListener('DOMContentLoaded', () => {
initSSE();
initOverclockChart();
fetchOverclockStatus();
setInterval(fetchOverclockStatus, 3000);
});
// ============ OVERCLOCK CONTROL ============
let ocProfiles = {};
let ocChart = null;
const OC_MAX_SAMPLES = 120;
function initOverclockChart() {
const canvas = document.getElementById('oc-chart');
if (!canvas || typeof Chart === 'undefined') return;
const ctx = canvas.getContext('2d');
ocChart = new Chart(ctx, {
type: 'line',
data: {
labels: [],
datasets: [
{ label: 'Core MHz', data: [], borderColor: '#22d3ee', backgroundColor: 'rgba(34,211,238,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: true },
{ label: 'Mem MHz', data: [], borderColor: '#c084fc', backgroundColor: 'rgba(192,132,252,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: false },
{ label: 'Temp °C', data: [], borderColor: '#fb7185', backgroundColor: 'rgba(251,113,133,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y1', fill: false },
{ label: 'Power W', data: [], borderColor: '#fbbf24', backgroundColor: 'rgba(251,191,36,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y2', fill: false },
{ label: 'Fan %', data: [], borderColor: '#34d399', backgroundColor: 'rgba(52,211,153,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y3', fill: false },
{ label: 'VRAM GB', data: [], borderColor: '#818cf8', backgroundColor: 'rgba(129,140,248,0.08)', borderWidth: 1.8, pointRadius: 0, tension: 0.35, yAxisID: 'y4', fill: false },
{ label: 'RAM Cache GB', data: [], borderColor: '#e879f9', backgroundColor: 'rgba(232,121,249,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y5', fill: false },
]
},
options: {
responsive: true,
maintainAspectRatio: false,
animation: false,
interaction: { mode: 'index', intersect: false },
scales: {
x: { ticks: { color: '#64748b', maxTicksLimit: 8, font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
y: { position: 'left', title: { display: true, text: 'MHz', color: '#22d3ee', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
y1: { position: 'right', title: { display: true, text: '°C', color: '#fb7185', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
y2: { position: 'right', offset: true, title: { display: true, text: 'W', color: '#fbbf24', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 400 },
y3: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
y4: { position: 'right', offset: true, title: { display: true, text: 'VRAM GB', color: '#818cf8', font: { size: 9 } }, ticks: { color: '#818cf8', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 16 },
y5: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 64 },
},
plugins: {
legend: { display: false },
tooltip: {
callbacks: {
label: function(context) {
const label = context.dataset.label || '';
const val = context.parsed.y;
if (label.includes('MHz')) return `${label}: ${val} MHz`;
if (label.includes('°C')) return `${label}: ${val}°C`;
if (label.includes('Power')) return `${label}: ${val} W`;
if (label.includes('Fan')) return `${label}: ${val}%`;
if (label.includes('GB')) return `${label}: ${val} GB`;
return `${label}: ${val}`;
}
}
}
},
}
});
}
function pushOverclockSample(gpu, ram) {
if (!ocChart || !gpu || !gpu.available) return;
const label = new Date().toLocaleTimeString([], { hour12: false });
ocChart.data.labels.push(label);
ocChart.data.datasets[0].data.push(gpu.clock_graphics_mhz || 0);
ocChart.data.datasets[1].data.push(gpu.clock_mem_mhz || 0);
ocChart.data.datasets[2].data.push(gpu.temperature_c || 0);
ocChart.data.datasets[3].data.push(gpu.power_w || 0);
ocChart.data.datasets[4].data.push(gpu.fan_pct || 0);
ocChart.data.datasets[5].data.push(parseFloat(gpu.vram_used_gb) || 0);
ocChart.data.datasets[6].data.push(parseFloat(ram?.cached_gb) || 0);
if (ocChart.data.labels.length > OC_MAX_SAMPLES) {
ocChart.data.labels.shift();
ocChart.data.datasets.forEach(d => d.data.shift());
}
ocChart.update('none');
}
let initialOcLoaded = false;
async function fetchOverclockStatus() {
try {
const resp = await fetch('/api/overclock');
if (!resp.ok) return;
const data = await resp.json();
ocProfiles = data.profiles || {};
renderOverclockStatus(data);
if (!initialOcLoaded && data.active_profile) {
initialOcLoaded = true;
const editSel = document.getElementById('oc-edit-profile');
if (editSel) {
editSel.value = data.active_profile;
loadOverclockForProfile(data.active_profile);
}
}
} catch (err) {
console.warn('Overclock fetch error:', err);
}
}
function renderOverclockStatus(data) {
const active = data.active_profile || 'balanced';
const badge = document.getElementById('oc-active-badge');
badge.textContent = `Active: ${active}`;
if (active === 'ollama') {
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-purple-950 border border-purple-700 text-purple-300';
} else if (active === 'comfy') {
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-cyan-950 border border-cyan-700 text-cyan-300';
} else {
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-300';
}
const gpu = data.gpu || {};
document.getElementById('oc-power-limit').textContent = `${gpu.power_limit_w ?? '--'} W`;
document.getElementById('oc-core').textContent = `${gpu.clock_sm_mhz ?? '--'} MHz`;
document.getElementById('oc-mem').textContent = `${gpu.clock_mem_mhz ?? '--'} MHz`;
document.getElementById('oc-temp-draw').textContent = `${gpu.temp_c ?? '--'}°C / ${gpu.power_draw_w ?? '--'}W`;
// Specs strip
document.getElementById('oc-spec-gpu').textContent = gpu.name || '--';
document.getElementById('oc-spec-driver').textContent = gpu.driver_version || '--';
const vramGb = (gpu.vram_total_mb || 0) / 1024;
document.getElementById('oc-spec-vram').textContent = vramGb > 0 ? vramGb.toFixed(0) + ' GB' : '--';
document.getElementById('oc-spec-maxcore').textContent = (gpu.clock_sm_max_mhz ?? '--') + ' MHz';
document.getElementById('oc-spec-maxmem').textContent = (gpu.clock_mem_max_mhz ?? '--') + ' MHz';
document.getElementById('oc-spec-power').textContent = `${gpu.power_limit_w ?? '--'} / ${gpu.power_max_w ?? '--'} W`;
// Fan status
const fan = data.fan || {};
const fanBadge = document.getElementById('oc-fan-badge');
if (fanBadge) {
if (fan.manual) {
fanBadge.textContent = `MANUAL (${fan.target_speed_pct ?? '--'}%)`;
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-500 text-emerald-300 font-bold';
} else {
fanBadge.textContent = 'AUTO (VBIOS)';
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-400 font-bold';
}
}
const xBadge = document.getElementById('oc-x-badge');
if (data.headless_x_running) {
xBadge.textContent = 'X: ON';
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-700 text-emerald-300';
} else {
xBadge.textContent = 'X: OFF';
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-rose-950 border border-rose-700 text-rose-300';
}
const offsets = (data.last_result && data.last_result.offsets) || {};
const note = document.getElementById('oc-offset-note');
if (offsets.supported) {
note.textContent = '✅ Clock offsets active (proprietary driver)';
note.className = 'text-[11px] font-mono text-emerald-400';
} else {
note.textContent = '⚠️ Clock offsets off — nvidia-open lacks offset support (power + clock locks active)';
note.className = 'text-[11px] font-mono text-amber-400';
}
// highlight active profile button
['ollama', 'comfy', 'balanced'].forEach(p => {
const btn = document.getElementById('oc-btn-' + p);
if (p === active) {
btn.classList.add('ring-2', 'ring-fuchsia-400');
} else {
btn.classList.remove('ring-2', 'ring-fuchsia-400');
}
});
}
function loadOverclockForProfile(name) {
const p = ocProfiles[name];
if (!p) return;
document.getElementById('oc-slider-power').value = p.power_limit_w || 370;
document.getElementById('oc-val-power').textContent = (p.power_limit_w || 370) + ' W';
document.getElementById('oc-slider-core').value = p.core_offset_mhz || 0;
document.getElementById('oc-val-core').textContent = '+' + (p.core_offset_mhz || 0) + ' MHz';
document.getElementById('oc-slider-mem').value = p.mem_offset_mhz || 0;
document.getElementById('oc-val-mem').textContent = '+' + (p.mem_offset_mhz || 0) + ' MHz';
document.getElementById('oc-lockcore-toggle').checked = (p.lock_core_max || 0) > 0;
document.getElementById('oc-val-lockcore').textContent = (p.lock_core_max || 0) > 0 ? 'On' : 'Off';
document.getElementById('oc-lockmem-toggle').checked = (p.lock_mem_mhz || 0) > 0;
document.getElementById('oc-val-lockmem').textContent = (p.lock_mem_mhz || 0) > 0 ? 'On' : 'Off';
const fanMode = p.fan_mode || 'auto';
const profFanMode = document.getElementById('oc-prof-fanmode');
if (profFanMode) profFanMode.value = fanMode;
const profFanSpeed = document.getElementById('oc-slider-prof-fanspeed');
if (profFanSpeed) profFanSpeed.value = p.fan_speed_pct || 70;
const profFanVal = document.getElementById('oc-val-prof-fanspeed');
if (profFanVal) profFanVal.textContent = (p.fan_speed_pct || 70) + '%';
toggleProfileFanMode();
}
function toggleProfileFanMode() {
const mode = document.getElementById('oc-prof-fanmode').value;
const container = document.getElementById('oc-prof-fanspeed-container');
const valBadge = document.getElementById('oc-val-prof-fanmode');
if (valBadge) valBadge.textContent = (mode === 'manual') ? 'Manual Target' : 'Auto (VBIOS)';
if (container) {
if (mode === 'manual') {
container.classList.remove('opacity-50', 'pointer-events-none');
} else {
container.classList.add('opacity-50', 'pointer-events-none');
}
}
}
function toggleCoreLock() {
document.getElementById('oc-val-lockcore').textContent = document.getElementById('oc-lockcore-toggle').checked ? 'On' : 'Off';
}
function toggleMemLock() {
document.getElementById('oc-val-lockmem').textContent = document.getElementById('oc-lockmem-toggle').checked ? 'On' : 'Off';
}
async function setFanManual(speed) {
let pct = speed;
if (pct === undefined || pct === null) {
pct = parseInt(document.getElementById('oc-fan-slider').value);
} else {
pct = parseInt(pct);
const slider = document.getElementById('oc-fan-slider');
if (slider) slider.value = pct;
const label = document.getElementById('oc-fan-slider-label');
if (label) label.textContent = pct + '%';
}
try {
const resp = await fetch('/api/overclock/fan', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ mode: 'manual', percent: pct })
});
const result = await resp.json();
if (!resp.ok) alert(`Fan set failed: ${result.detail || 'error'}`);
await fetchOverclockStatus();
} catch (err) {
alert(`Error setting fan speed: ${err}`);
}
}
async function setFanAuto() {
try {
const resp = await fetch('/api/overclock/fan', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ mode: 'auto' })
});
const result = await resp.json();
if (!resp.ok) alert(`Fan auto failed: ${result.detail || 'error'}`);
await fetchOverclockStatus();
} catch (err) {
alert(`Error setting fan to auto: ${err}`);
}
}
async function applyOverclock(profile) {
try {
const resp = await fetch('/api/overclock/apply', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ profile })
});
const result = await resp.json();
if (!resp.ok) {
alert(`Apply failed: ${result.detail || 'error'}`);
}
const editSel = document.getElementById('oc-edit-profile');
if (editSel) {
editSel.value = profile;
loadOverclockForProfile(profile);
}
await fetchOverclockStatus();
} catch (err) {
alert(`Error: ${err}`);
}
}
async function saveOverclockProfile() {
const name = document.getElementById('oc-edit-profile').value;
const power = parseInt(document.getElementById('oc-slider-power').value);
const core = parseInt(document.getElementById('oc-slider-core').value);
const mem = parseInt(document.getElementById('oc-slider-mem').value);
const lockCore = document.getElementById('oc-lockcore-toggle').checked;
const lockMem = document.getElementById('oc-lockmem-toggle').checked;
const fanMode = document.getElementById('oc-prof-fanmode') ? document.getElementById('oc-prof-fanmode').value : 'auto';
const fanSpeed = document.getElementById('oc-slider-prof-fanspeed') ? parseInt(document.getElementById('oc-slider-prof-fanspeed').value) : 70;
const config = {
power_limit_w: power,
core_offset_mhz: core,
mem_offset_mhz: mem,
lock_core_min: lockCore ? 2900 : 0,
lock_core_max: lockCore ? 3105 : 0,
lock_mem_mhz: lockMem ? 11501 : 0,
fan_mode: fanMode,
fan_speed_pct: (fanMode === 'manual') ? fanSpeed : 0,
};
try {
const resp = await fetch(`/api/overclock/profiles/${name}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ config })
});
const result = await resp.json();
if (resp.ok) {
await fetchOverclockStatus();
await applyOverclock(name);
} else {
alert(`Save failed: ${result.detail || 'error'}`);
}
} catch (err) {
alert(`Error: ${err}`);
}
}
// ============================================================================
// THERMAL GOVERNOR / RESIDENCY / ANALYTICS / AUTOTUNE
// ============================================================================
let governorEnabled = true;
function renderGovernor(gov) {
if (!gov) return;
governorEnabled = gov.enabled;
const levels = 3;
const el = (id) => document.getElementById(id);
if (!el('gov-label')) return;
el('gov-label').textContent = gov.label || '—';
el('gov-label').className = 'text-2xl font-bold ' +
(gov.level === 0 ? 'text-emerald-400' : gov.level < 3 ? 'text-amber-400' : 'text-rose-400');
el('gov-level').textContent = `level ${gov.level} / ${levels}`;
el('gov-bar').style.width = `${(gov.level / levels) * 100}%`;
el('gov-esc').textContent = `${gov.escalate_at_c}°C`;
el('gov-rec').textContent = `${gov.recover_below_c}°C`;
el('gov-scale').textContent = `${Math.round((gov.offset_scale ?? 1) * 100)}%`;
el('gov-reason').textContent = gov.last_reason || '—';
const toggle = el('gov-toggle');
toggle.textContent = gov.enabled ? 'Enabled' : 'Disabled';
toggle.className = 'px-2.5 py-1 text-xs font-semibold rounded-lg border transition ' +
(gov.enabled ? 'bg-emerald-950/70 border-emerald-800 text-emerald-300 hover:bg-emerald-900'
: 'bg-slate-800 border-slate-700 text-slate-400 hover:bg-slate-700');
const hist = el('gov-history');
hist.innerHTML = (gov.history || []).map(h => {
const t = new Date(h.ts * 1000).toLocaleTimeString();
const up = h.to_level > h.from_level;
return `
${t} ${up ? '▲' : '▼'} L${h.to_level} ${h.label}
${h.reason || ''}
`;
}).join('') || 'No derates recorded — the card has stayed happy.
';
}
async function toggleGovernor() {
const res = await fetch('/api/governor', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ enabled: !governorEnabled })
});
renderGovernor(await res.json());
}
async function resetGovernor() {
const res = await fetch('/api/governor', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ reset: true })
});
renderGovernor(await res.json());
}
// ---------------------------------------------------------------- residency
async function fetchCacheReport(refresh = false) {
const body = document.getElementById('cache-files');
if (!body) return;
if (refresh) body.innerHTML = 'Scanning…
';
try {
const res = await fetch(`/api/cache/report?refresh=${refresh ? 'true' : 'false'}`);
const d = await res.json();
document.getElementById('cache-total').textContent =
`${d.resident_total_gb} / ${d.catalog_total_gb} GB`;
document.getElementById('cache-meta').textContent =
`${d.residency_pct}% resident · ${d.warm_files}/${d.files_scanned} warm · ${d.scan_ms}ms`;
const cap = d.capability || {};
document.getElementById('cache-capability').innerHTML = cap.exact_everywhere
? 'Exact measurement available for every model file. '
: `Ollama weights measured by read-rate probe — ${cap.reason || ''}`;
body.innerHTML = (d.files || []).slice(0, 40).map(f => {
const pct = f.resident_pct ?? 0;
const color = pct >= 90 ? 'bg-emerald-500' : pct >= 25 ? 'bg-amber-500' : 'bg-slate-700';
const kind = f.kind === 'ollama' ? 'text-cyan-400' : 'text-fuchsia-400';
return `
${f.name}
${pct}% · ${f.size_gb}GB
`;
}).join('');
} catch (e) {
body.innerHTML = `Residency scan failed: ${e}
`;
}
}
async function fetchWarmPlan() {
const body = document.getElementById('cache-files');
body.innerHTML = 'Building plan…
';
try {
const d = await (await fetch('/api/warm-plan')).json();
document.getElementById('cache-total').textContent = `${d.planned_gb} GB planned`;
document.getElementById('cache-meta').textContent =
`budget ${d.budget_gb} GB · ${d.warm_count} to warm · ${d.skipped_count} skipped`;
document.getElementById('cache-capability').innerHTML =
'Ranked by recency/frequency; already-resident files cost nothing. ';
body.innerHTML = d.plan.map((p, i) => `
${i + 1}. ${p.name}
${p.size_gb}GB · ${p.resident_pct}% · s=${p.score}
`).join('') || 'Everything worth warming is already resident.
';
} catch (e) {
body.innerHTML = `${e}
`;
}
}
// ---------------------------------------------------------------- analytics
async function fetchAnalytics() {
const el = document.getElementById('analytics-body');
if (!el) return;
const days = document.getElementById('analytics-days').value;
try {
const d = await (await fetch(`/api/analytics/profiles?days=${days}`)).json();
if (!d.profiles || !d.profiles.length) {
el.innerHTML = 'No swaps recorded yet in this window. Switch a model to start building the comparison.
';
return;
}
const best = Math.max(...d.profiles.map(p => p.avg_tok_s || 0));
el.innerHTML = `
profile tok/s load GB/s °C avg W avg SM MHz n
` +
d.profiles.map(p => {
const win = (p.avg_tok_s || 0) === best;
return `
${win ? '★ ' : ''}${p.profile ?? '—'}
${p.avg_tok_s ?? '—'} ${p.avg_load_gbps ?? '—'}
${p.avg_temp_c ?? '—'} ${p.avg_power_w ?? '—'}
${p.avg_clock_sm ?? '—'} ${p.swaps} `;
}).join('') + '
';
} catch (e) {
el.innerHTML = `${e}
`;
}
}
// ---------------------------------------------------------------- autotune
let sweepPoll = null;
async function runSweep() {
const btn = document.getElementById('at-run');
const body = document.getElementById('autotune-body');
const payload = {
knob: document.getElementById('at-knob').value,
profile: document.getElementById('at-profile').value,
start: parseInt(document.getElementById('at-start').value, 10),
stop: parseInt(document.getElementById('at-stop').value, 10),
step: parseInt(document.getElementById('at-step').value, 10),
apply_best: document.getElementById('at-apply').checked,
};
btn.disabled = true;
btn.classList.add('opacity-50');
body.innerHTML = 'Sweeping… each step loads the model and benchmarks decode. This takes a few minutes.
';
sweepPoll = setInterval(pollSweep, 2000);
try {
const res = await fetch('/api/autotune/sweep', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
});
const d = await res.json();
renderSweep(res.ok ? d : { error: d.detail });
} catch (e) {
body.innerHTML = `${e}
`;
} finally {
clearInterval(sweepPoll);
btn.disabled = false;
btn.classList.remove('opacity-50');
document.getElementById('autotune-state').textContent = 'idle';
}
}
async function pollSweep() {
try {
const d = await (await fetch('/api/autotune')).json();
const s = document.getElementById('autotune-state');
if (d.running && d.current) {
s.textContent = `${d.current.knob}=${d.current.value} → ${d.current.tokens_per_sec} tok/s`;
s.className = 'text-xs font-mono text-fuchsia-400';
}
} catch (e) { /* transient */ }
}
async function cancelSweep() {
await fetch('/api/autotune/cancel', { method: 'POST' });
document.getElementById('autotune-state').textContent = 'cancelling…';
}
function renderSweep(d) {
const body = document.getElementById('autotune-body');
if (d.error) { body.innerHTML = `${d.error}
`; return; }
const rows = (d.table || []).map(r => `
${r.value} ${r.tokens_per_sec} ${r.temp_c}
${r.power_w ?? '—'} ${r.clock_mem_mhz ?? '—'}
${r.instability || 'stable'} `).join('');
const best = d.best
? `★ Best stable: ${d.knob}=${d.best.value}
→ ${d.best.tokens_per_sec} tok/s at ${d.best.temp_c}°C
${d.gain_vs_current_pct != null
? `(${d.gain_vs_current_pct > 0 ? '+' : ''}${d.gain_vs_current_pct}% vs your current setting) `
: d.gain_vs_first_step_pct != null
? `(spread across tried values: ${d.gain_vs_first_step_pct > 0 ? '+' : ''}${d.gain_vs_first_step_pct}% vs slowest) `
: ''}
${d.applied_to_profile ? '· saved to profile ' : ''}
`
: 'No stable step produced throughput.
';
const unstable = d.first_unstable
? `Stopped climbing at ${d.first_unstable.value}: ${d.first_unstable.why}
` : '';
body.innerHTML = best + unstable + `
offset tok/s °C W mem MHz status
${rows}
`;
}
// ---------------------------------------------------------------- wiring
document.addEventListener('DOMContentLoaded', () => {
fetchCacheReport(false);
fetchAnalytics();
setInterval(fetchAnalytics, 60000);
setInterval(() => fetchCacheReport(false), 30000);
fetch('/api/autotune').then(r => r.json()).then(d => {
if (d.last_result) renderSweep(d.last_result);
}).catch(() => {});
});
// ---------------------------------------------------------------- arbitration
function renderArbitrator(arb, gpu) {
const el = (id) => document.getElementById(id);
if (!el('arb-action')) return;
const c = arb.counters || {};
// VRAM held by processes HyperSwap cannot reclaim. Worth showing: it is headroom the
// arbitrator can never give back, no matter how much it purges.
const bd = (gpu && gpu.breakdown) || {};
const un = el('arb-unmanaged');
if (un) {
const procs = bd.unmanaged || [];
un.innerHTML = procs.length
? `${bd.unmanaged_gb} GB unreclaimable — ` +
procs.map(p => `${p.name} (${p.vram_mb} MB)`).join(', ')
: '';
}
el('arb-action').textContent = arb.last_action || 'Idle';
el('arb-yields').textContent = c.yields ?? 0;
el('arb-busy').textContent = c.yield_deferred_busy ?? 0;
el('arb-later').textContent = c.deferred_releases ?? 0;
el('arb-stalled').textContent = c.yield_stalled ?? 0;
el('arb-purges').textContent = c.purges ?? 0;
el('arb-defpurge').textContent = c.deferred_purges ?? 0;
const ws = el('arb-ws');
ws.textContent = arb.connected_ws ? 'ComfyUI WS live' : 'WS down — polling';
ws.className = 'text-xs font-mono ' + (arb.connected_ws ? 'text-emerald-400' : 'text-amber-400');
// Show why we are holding off, and the idle countdown before ComfyUI is purged.
const parts = [];
const backoff = arb.yield_backoff || {};
for (const [model, secs] of Object.entries(backoff)) {
parts.push(`waiting ${secs}s before asking '${model}' again`);
}
if (arb.pending_purge && arb.comfy_idle_s != null) {
const left = Math.max((arb.idle_purge_after_s || 0) - arb.comfy_idle_s, 0).toFixed(0);
parts.push(`ComfyUI idle ${arb.comfy_idle_s}s — holding its checkpoints ${left}s longer`);
}
el('arb-backoff').textContent = parts.join(' · ');
}
// ---------------------------------------------------------------- health
async function fetchHealth(verbose = false) {
const badge = document.getElementById('health-badge');
const body = document.getElementById('health-body');
if (!badge) return;
try {
const d = await (await fetch('/api/health')).json();
const style = {
ok: 'bg-emerald-950/70 border border-emerald-800 text-emerald-300',
degraded: 'bg-amber-950/70 border border-amber-800 text-amber-300',
failed: 'bg-rose-950/70 border border-rose-800 text-rose-300',
}[d.status];
badge.className = `px-2 py-1 text-xs font-bold rounded-lg ${style}`;
badge.textContent = `${d.status.toUpperCase()} · ${d.summary}`;
// Healthy checks collapse to one line; anything wrong is shown in full with the
// impact and the fix, because that is the information you actually need.
const problems = d.checks.filter(c => c.status !== 'ok');
const shown = (verbose || problems.length) ? (verbose ? d.checks : problems) : [];
body.innerHTML = shown.map(c => {
const colour = c.status === 'ok' ? 'text-emerald-400'
: c.status === 'degraded' ? 'text-amber-400' : 'text-rose-400';
const mark = c.status === 'ok' ? '✓' : c.status === 'degraded' ? '!' : '✗';
let html = `${mark} ${c.name} ` +
` — ${c.detail}
`;
if (c.status !== 'ok') {
if (c.impact) html += `impact: ${c.impact}
`;
if (c.fix) html += `fix: ${c.fix}
`;
}
return html;
}).join('') || 'All dependencies healthy.
';
} catch (e) {
badge.className = 'px-2 py-1 text-xs font-bold rounded-lg bg-rose-950/70 border border-rose-800 text-rose-300';
badge.textContent = 'UNREACHABLE';
body.innerHTML = `${e}
`;
}
}
document.addEventListener('DOMContentLoaded', () => {
fetchHealth(false);
setInterval(() => fetchHealth(false), 60000);
});
// ---------------------------------------------------------------- honest labels
async function fetchMeasuredResidency() {
const el = document.getElementById('legend-models-resident');
if (!el) return;
try {
const d = await (await fetch('/api/cache/report?files=false')).json();
// Deliberately distinct from the page-cache figure beside it: that is every file
// the kernel has cached; this is model weights, measured.
el.textContent = `${d.resident_total_gb} GB of ${d.catalog_total_gb} GB`;
const cr = document.getElementById('comfy-resident');
if (cr) cr.textContent = `${d.residency_pct}%`;
} catch (e) {
el.textContent = 'unavailable';
}
}
async function fetchVersion() {
const el = document.getElementById('app-version');
if (!el) return;
try {
const d = await (await fetch('/openapi.json')).json();
el.textContent = `v${d.info.version}`;
} catch (e) {
el.textContent = 'v?';
}
}
document.addEventListener('DOMContentLoaded', () => {
fetchVersion();
fetchMeasuredResidency();
setInterval(fetchMeasuredResidency, 30000);
});
// ---------------------------------------------------------------- durable swap stats
let lastSwapFromStore = null;
async function fetchLastSwapFromStore() {
// The in-memory ring empties on restart, which blanked the swap tiles even though
// every swap is on disk. Fall back to the persisted log so a restart does not erase
// what the dashboard knows.
try {
const rows = await (await fetch('/api/history?durable=true&limit=50')).json();
const swap = (rows || []).find(r => r.event_type === 'LLM Model Switch');
if (!swap) return;
lastSwapFromStore = swap;
const t = document.getElementById('ollama-last-swap');
const c = document.getElementById('ollama-cache-hit');
if (t && (!t.textContent || t.textContent.includes('\u2014') || t.textContent.includes('--'))) {
t.textContent = `${Math.round(swap.duration_ms)} ms`;
}
if (c && (!c.textContent || c.textContent.includes('\u2014') || c.textContent.includes('--')
|| c.textContent === 'no swaps yet')) {
c.textContent = swap.cache_status || 'OK';
}
} catch (e) { /* the live stream is the primary source; this is only a fallback */ }
}
document.addEventListener('DOMContentLoaded', () => {
setTimeout(fetchLastSwapFromStore, 1500);
});
// ---------------------------------------------------------------- engine config
async function fetchEngineConfig() {
// These subtitles used to be hardcoded. They happened to be accurate, which is worse
// than being wrong: they would have stayed accurate-looking after the settings changed.
try {
const d = await (await fetch('/api/engines')).json();
const o = document.getElementById('ollama-engine-sub');
if (o && d.ollama) {
const bits = [`Port :${d.ollama.port}`, d.ollama.summary];
if (d.ollama.max_loaded_models) bits.push(`${d.ollama.max_loaded_models} model resident`);
if (d.ollama.keep_alive) bits.push(`keep-alive ${d.ollama.keep_alive}`);
o.textContent = bits.join(' // ');
o.title = (d.ollama.settings || [])
.filter(s => s.means)
.map(s => `${s.key}=${s.value} — ${s.means}`)
.join('\n');
}
const c = document.getElementById('comfy-engine-sub');
if (c && d.comfyui && d.comfyui.online) {
c.textContent = `Port :8188 // v${d.comfyui.version} // ${d.comfyui.summary}`;
c.title = `torch ${d.comfyui.pytorch}\n${d.comfyui.device || ''}`;
}
} catch (e) { /* subtitles are cosmetic; never break the page over them */ }
}
document.addEventListener('DOMContentLoaded', () => {
fetchEngineConfig();
setInterval(fetchEngineConfig, 120000);
});