fb104ac9c0
Self-check. Fan control failed for an entire session -- recoverably, and completely invisibly. It appeared once, inside one field of one log line, and nothing ever asked whether fan control worked. health.py now checks everything this service depends on (NVML, passwordless sudo for nvidia-smi, fan control via the headless X server, overclock drift, the telemetry store, residency measurement capability, model directories, the ComfyUI websocket, and both upstream HTTP services) and reports for each one what is broken, what that breaks, and how to fix it. Exposed at GET /api/health, as an MCP tool, and as a dashboard panel that collapses to a badge when healthy and expands to impact-and-fix when not. Current state: 9 ok, 1 degraded (the known cachestat permission limit on Ollama's blobs). A self-check that returns ok while a dependency is broken is worse than none, so the tests drive each check to its failure state -- including the exact "Error resolving target specification 'gpu:0'" string from the original incident -- and assert that a check which raises surfaces as failed rather than taking down the endpoint. SSE payload. The installed-model catalog was 10.6 KB of a 13.1 KB frame, 81% of the stream, re-sent to every subscriber every second despite changing only when a model is pulled or removed: 135 MB/hour across three tabs. It is now sent on a subscriber's first frame and whenever the set changes; the client keeps the last known list. Steady-state frames dropped from 14041 to 3664 bytes, a 74% reduction, and /api/stats still returns the complete snapshot for API consumers. Tests: 182 (was 169). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
979 lines
44 KiB
JavaScript
979 lines
44 KiB
JavaScript
let evtSource = null;
|
|
let currentInstalledModels = [];
|
|
|
|
// Initialize SSE Stream
|
|
function initSSE() {
|
|
if (evtSource) {
|
|
evtSource.close();
|
|
}
|
|
|
|
evtSource = new EventSource('/api/stream');
|
|
|
|
evtSource.onopen = () => {
|
|
document.getElementById('sse-badge').textContent = 'SSE LIVE';
|
|
document.getElementById('sse-badge').className = 'text-emerald-400 font-bold';
|
|
};
|
|
|
|
evtSource.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
updateDashboard(data);
|
|
} catch (err) {
|
|
console.error('Error parsing SSE event:', err);
|
|
}
|
|
};
|
|
|
|
evtSource.onerror = (err) => {
|
|
console.warn('SSE disconnected, retrying...', err);
|
|
document.getElementById('sse-badge').textContent = 'RECONNECTING...';
|
|
document.getElementById('sse-badge').className = 'text-rose-400 font-bold';
|
|
};
|
|
}
|
|
|
|
function updateDashboard(data) {
|
|
if (!data) return;
|
|
|
|
// Governor and arbitration state ride along in the shared snapshot.
|
|
if (data.governor) renderGovernor(data.governor);
|
|
if (data.arbitrator) renderArbitrator(data.arbitrator, data.gpu);
|
|
|
|
// 1. GPU VRAM Stats
|
|
const gpu = data.gpu || {};
|
|
const ram = data.ram || {};
|
|
if (gpu.available) {
|
|
pushOverclockSample(gpu, ram);
|
|
document.getElementById('gpu-chip-name').textContent = gpu.device_name || 'NVIDIA GPU';
|
|
document.getElementById('vram-total-used').textContent = gpu.vram_used_gb || '0.0';
|
|
document.getElementById('vram-used-pct').textContent = `${gpu.vram_used_pct || 0}% USED`;
|
|
|
|
const bd = gpu.breakdown || {};
|
|
const totalBytes = gpu.vram_total_bytes || (16 * 1024**3);
|
|
|
|
const ollamaPct = ((bd.ollama_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const comfyPct = ((bd.comfyui_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const systemPct = ((bd.system_gb || 0) / (gpu.vram_total_gb || 16)) * 100;
|
|
const freePct = Math.max(0, 100 - (ollamaPct + comfyPct + systemPct));
|
|
|
|
document.getElementById('bar-ollama').style.width = `${ollamaPct}%`;
|
|
document.getElementById('bar-comfy').style.width = `${comfyPct}%`;
|
|
document.getElementById('bar-system').style.width = `${systemPct}%`;
|
|
document.getElementById('bar-free').style.width = `${freePct}%`;
|
|
|
|
document.getElementById('tooltip-ollama').textContent = `${bd.ollama_gb || 0} GB`;
|
|
document.getElementById('tooltip-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
|
|
document.getElementById('tooltip-system').textContent = `${bd.system_gb || 0} GB`;
|
|
|
|
document.getElementById('legend-ollama').textContent = `${bd.ollama_gb || 0} GB`;
|
|
document.getElementById('legend-comfy').textContent = `${bd.comfyui_gb || 0} GB`;
|
|
document.getElementById('legend-system').textContent = `${bd.system_gb || 0} GB`;
|
|
document.getElementById('legend-free').textContent = `${bd.free_gb || 0} GB`;
|
|
|
|
// Hardware sensors
|
|
document.getElementById('gpu-util-val').textContent = `${gpu.gpu_util_pct || 0}%`;
|
|
document.getElementById('gpu-temp-val').textContent = `${gpu.temperature_c || 0}°C`;
|
|
document.getElementById('gpu-power-val').textContent = `${gpu.power_w || 0} W`;
|
|
document.getElementById('gpu-fan-val').textContent = `${gpu.fan_pct || 0}%`;
|
|
|
|
// Per-fan and animations
|
|
const fan0 = (gpu.fans && gpu.fans.length > 0) ? gpu.fans[0] : (gpu.fan_pct || 0);
|
|
const fan1 = (gpu.fans && gpu.fans.length > 1) ? gpu.fans[1] : (gpu.fan_pct || 0);
|
|
const fanSub = document.getElementById('gpu-fan-sub');
|
|
if (fanSub) fanSub.textContent = `Fan 0: ${fan0}% | Fan 1: ${fan1}%`;
|
|
|
|
const fan0Val = document.getElementById('oc-fan0-val');
|
|
if (fan0Val) fan0Val.textContent = `${fan0}%`;
|
|
const fan0Bar = document.getElementById('oc-fan0-bar');
|
|
if (fan0Bar) fan0Bar.style.width = `${fan0}%`;
|
|
|
|
const fan1Val = document.getElementById('oc-fan1-val');
|
|
if (fan1Val) fan1Val.textContent = `${fan1}%`;
|
|
const fan1Bar = document.getElementById('oc-fan1-bar');
|
|
if (fan1Bar) fan1Bar.style.width = `${fan1}%`;
|
|
|
|
const fanCurrent = document.getElementById('oc-fan-current');
|
|
if (fanCurrent) fanCurrent.textContent = `${gpu.fan_pct || 0}%`;
|
|
|
|
const spinSpeed = Math.max(0.2, (100 - (gpu.fan_pct || 0)) / 100 * 1.6 + 0.3);
|
|
const fanIcon = document.getElementById('gpu-fan-icon');
|
|
if (fanIcon) {
|
|
if ((gpu.fan_pct || 0) > 0) {
|
|
fanIcon.classList.add('fan-spinning');
|
|
fanIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
|
|
} else {
|
|
fanIcon.classList.remove('fan-spinning');
|
|
}
|
|
}
|
|
const ocFanCardIcon = document.getElementById('oc-fan-card-icon');
|
|
if (ocFanCardIcon) {
|
|
if ((gpu.fan_pct || 0) > 0) {
|
|
ocFanCardIcon.classList.add('fan-spinning');
|
|
ocFanCardIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`;
|
|
} else {
|
|
ocFanCardIcon.classList.remove('fan-spinning');
|
|
}
|
|
}
|
|
|
|
// Processes table
|
|
const tbody = document.getElementById('gpu-proc-table');
|
|
if (bd.processes && bd.processes.length > 0) {
|
|
tbody.innerHTML = bd.processes.map(p => {
|
|
let tagClass = 'text-slate-400';
|
|
let badge = '';
|
|
if (p.is_ollama) {
|
|
tagClass = 'text-purple-400 font-bold';
|
|
badge = '<span class="px-1 py-0.2 bg-purple-950 text-purple-300 rounded border border-purple-800 text-[9px] mr-1">OLLAMA</span>';
|
|
} else if (p.is_comfy) {
|
|
tagClass = 'text-cyan-400 font-bold';
|
|
badge = '<span class="px-1 py-0.2 bg-cyan-950 text-cyan-300 rounded border border-cyan-800 text-[9px] mr-1">COMFY</span>';
|
|
}
|
|
return `
|
|
<tr class="hover:bg-slate-900/50">
|
|
<td class="py-1 text-slate-500 font-mono text-[10px]">${p.pid}</td>
|
|
<td class="py-1 ${tagClass} text-[11px] truncate max-w-[140px]">${badge}${p.name}</td>
|
|
<td class="py-1 text-right font-mono font-bold text-slate-200 text-[11px]">${p.vram_mb} MB</td>
|
|
</tr>
|
|
`;
|
|
}).join('');
|
|
} else {
|
|
tbody.innerHTML = '<tr><td colspan="3" class="py-2 text-center text-slate-500">No compute processes running</td></tr>';
|
|
}
|
|
}
|
|
|
|
// 2. System RAM & Page Cache
|
|
if (ram.total_bytes) {
|
|
document.getElementById('ram-cached-gb').textContent = ram.cached_gb || '0.0';
|
|
document.getElementById('ram-total-text').textContent = `${ram.total_gb || 0} GB Total (${ram.cache_ratio_pct || 0}% in Cache)`;
|
|
|
|
const usedPct = ((ram.used_bytes || 0) / ram.total_bytes) * 100;
|
|
const cachePct = ((ram.cached_bytes || 0) / ram.total_bytes) * 100;
|
|
const freePct = Math.max(0, 100 - (usedPct + cachePct));
|
|
|
|
document.getElementById('bar-ram-used').style.width = `${usedPct}%`;
|
|
document.getElementById('bar-ram-cache').style.width = `${cachePct}%`;
|
|
document.getElementById('bar-ram-free').style.width = `${freePct}%`;
|
|
|
|
document.getElementById('legend-ram-used').textContent = `${ram.used_gb || 0} GB`;
|
|
document.getElementById('legend-ram-cached').textContent = `${ram.cached_gb || 0} GB`;
|
|
document.getElementById('legend-ram-free').textContent = `${ram.free_gb || 0} GB`;
|
|
}
|
|
|
|
// 3. Ollama State
|
|
const ollama = data.ollama || {};
|
|
if (ollama.online) {
|
|
document.getElementById('ollama-status-text').textContent = 'ONLINE';
|
|
document.getElementById('ollama-status-text').className = 'text-emerald-400 font-mono font-bold';
|
|
document.getElementById('ollama-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-emerald-500';
|
|
document.getElementById('ollama-pulse').className = 'animate-ping absolute inline-flex h-full w-full rounded-full bg-emerald-400 opacity-75';
|
|
|
|
if (ollama.active_model_name) {
|
|
document.getElementById('ollama-active-model').textContent = ollama.active_model_name;
|
|
document.getElementById('ollama-vram-badge').textContent = `${ollama.active_model_vram_gb || 0} GB VRAM`;
|
|
document.getElementById('ollama-vram-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-purple-900/80 text-purple-200 border border-purple-600 font-bold';
|
|
document.getElementById('ollama-context').textContent = `${ollama.active_context || 0} ctx`;
|
|
} else {
|
|
document.getElementById('ollama-active-model').textContent = 'None Loaded (VRAM Free)';
|
|
document.getElementById('ollama-vram-badge').textContent = '0.0 GB VRAM';
|
|
document.getElementById('ollama-vram-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-slate-800 text-slate-400 border border-slate-700';
|
|
document.getElementById('ollama-context').textContent = 'Idle';
|
|
}
|
|
|
|
// The catalog is omitted from stream frames when unchanged, so keep the last one.
|
|
if (ollama.installed_models && ollama.installed_models.length > 0) {
|
|
currentInstalledModels = ollama.installed_models;
|
|
}
|
|
if (currentInstalledModels && currentInstalledModels.length > 0) {
|
|
document.getElementById('ollama-total-models').textContent = currentInstalledModels.length;
|
|
updateModelSelect(currentInstalledModels, ollama.active_model_name);
|
|
}
|
|
} else {
|
|
document.getElementById('ollama-status-text').textContent = 'OFFLINE';
|
|
document.getElementById('ollama-status-text').className = 'text-rose-400 font-mono font-bold';
|
|
document.getElementById('ollama-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-rose-500';
|
|
document.getElementById('ollama-pulse').className = 'hidden';
|
|
}
|
|
|
|
// 4. ComfyUI State
|
|
const comfy = data.comfyui || {};
|
|
if (comfy.online) {
|
|
document.getElementById('comfy-status-text').textContent = 'ONLINE';
|
|
document.getElementById('comfy-status-text').className = 'text-emerald-400 font-mono font-bold';
|
|
document.getElementById('comfy-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-emerald-500';
|
|
document.getElementById('comfy-pulse').className = 'animate-ping absolute inline-flex h-full w-full rounded-full bg-emerald-400 opacity-75';
|
|
|
|
document.getElementById('comfy-queue-count').textContent = comfy.queue_running + comfy.queue_remaining;
|
|
if (comfy.executing) {
|
|
document.getElementById('comfy-exec-badge').textContent = 'GENERATING...';
|
|
document.getElementById('comfy-exec-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-amber-950 text-amber-300 border border-amber-600 animate-pulse font-bold';
|
|
document.getElementById('comfy-model-info').textContent = `Prompt #${comfy.current_prompt_id || 'Active'}`;
|
|
} else {
|
|
document.getElementById('comfy-exec-badge').textContent = 'IDLE / READY';
|
|
document.getElementById('comfy-exec-badge').className = 'px-2 py-0.5 text-[10px] font-mono rounded bg-emerald-950 text-emerald-300 border border-emerald-800';
|
|
document.getElementById('comfy-model-info').textContent = 'Dynamic Model Offloader';
|
|
}
|
|
|
|
document.getElementById('comfy-vram-avail').textContent = `${(comfy.vram_free_mb / 1024).toFixed(1)} GB`;
|
|
} else {
|
|
document.getElementById('comfy-status-text').textContent = 'OFFLINE';
|
|
document.getElementById('comfy-status-text').className = 'text-rose-400 font-mono font-bold';
|
|
document.getElementById('comfy-dot').className = 'relative inline-flex rounded-full h-2.5 w-2.5 bg-rose-500';
|
|
document.getElementById('comfy-pulse').className = 'hidden';
|
|
}
|
|
|
|
document.getElementById('comfy-discovered-count').textContent = `${data.comfy_models_count || 0} Files`;
|
|
|
|
// 5. Switch History Timeline
|
|
const history = data.history || [];
|
|
const logContainer = document.getElementById('switch-log-container');
|
|
if (history.length > 0) {
|
|
const latest = history[0];
|
|
document.getElementById('ollama-last-swap').textContent = `${latest.duration_ms} ms`;
|
|
document.getElementById('ollama-cache-hit').textContent = latest.cache_status || 'OK';
|
|
|
|
logContainer.innerHTML = history.slice(0, 10).map(item => {
|
|
const isHit = (item.cache_status || '').includes('RAM Cache Hit') || (item.cache_status || '').includes('RAM-Cached');
|
|
const badgeClass = isHit
|
|
? 'bg-emerald-950/80 text-emerald-300 border-emerald-800'
|
|
: 'bg-amber-950/80 text-amber-300 border-amber-800';
|
|
return `
|
|
<div class="p-2.5 rounded-xl bg-slate-950 border border-slate-800 text-xs font-mono flex items-center justify-between">
|
|
<div class="space-y-0.5">
|
|
<div class="flex items-center space-x-1.5">
|
|
<span class="text-slate-500 text-[10px]">${item.timestamp}</span>
|
|
<span class="text-indigo-400 font-bold">${item.event_type}</span>
|
|
</div>
|
|
<div class="text-[11px] text-slate-300 truncate max-w-[260px]">
|
|
<span class="text-slate-500">${item.source}</span> → <span class="text-cyan-300 font-bold">${item.target}</span>
|
|
</div>
|
|
</div>
|
|
<div class="text-right">
|
|
<div class="text-slate-100 font-bold">${item.duration_ms} ms</div>
|
|
<span class="px-1.5 py-0.2 text-[9px] rounded border ${badgeClass}">${item.cache_status}</span>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
}
|
|
}
|
|
|
|
function updateModelSelect(models, activeModel) {
|
|
const select = document.getElementById('ollama-model-select');
|
|
const currentVal = select.value;
|
|
|
|
if (JSON.stringify(models.map(m => m.name)) === JSON.stringify(currentInstalledModels)) {
|
|
return;
|
|
}
|
|
|
|
currentInstalledModels = models.map(m => m.name);
|
|
select.innerHTML = models.map(m => {
|
|
const isSelected = m.name === activeModel || m.name === currentVal;
|
|
const sizeGb = (m.size / (1024**3)).toFixed(1);
|
|
const quant = m.details?.quantization_level || '';
|
|
return `<option value="${m.name}" ${isSelected ? 'selected' : ''}>${m.name} (${sizeGb} GB ${quant})</option>`;
|
|
}).join('');
|
|
}
|
|
|
|
// User Actions
|
|
async function triggerModelSwitch() {
|
|
const select = document.getElementById('ollama-model-select');
|
|
const targetModel = select.value;
|
|
if (!targetModel) return;
|
|
|
|
const btn = document.getElementById('btn-switch-model');
|
|
btn.disabled = true;
|
|
btn.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i> <span>Swapping...</span>';
|
|
|
|
try {
|
|
const resp = await fetch('/api/switch-model', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ model: targetModel, keep_alive: '30m' })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) {
|
|
alert(`Switch failed: ${result.detail || 'Unknown error'}`);
|
|
}
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
} finally {
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="fa-solid fa-shuffle"></i> <span>Hot Swap</span>';
|
|
}
|
|
}
|
|
|
|
async function freeOllamaVRAM() {
|
|
try {
|
|
const resp = await fetch('/api/free-vram', { method: 'POST' });
|
|
const data = await resp.json();
|
|
console.log('Ollama VRAM yielded:', data);
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function freeComfyVRAM() {
|
|
try {
|
|
const resp = await fetch('/api/comfy-free', { method: 'POST' });
|
|
const data = await resp.json();
|
|
console.log('ComfyUI VRAM freed:', data);
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function warmAllModels() {
|
|
try {
|
|
const resp = await fetch('/api/warm-all', { method: 'POST' });
|
|
const data = await resp.json();
|
|
alert(`Warming completed in ${data.total_duration_ms} ms! All models are now cached in 64GB RAM.`);
|
|
} catch (err) {
|
|
alert(`Error warming models: ${err}`);
|
|
}
|
|
}
|
|
|
|
// Startup
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
initSSE();
|
|
initOverclockChart();
|
|
fetchOverclockStatus();
|
|
setInterval(fetchOverclockStatus, 3000);
|
|
});
|
|
|
|
// ============ OVERCLOCK CONTROL ============
|
|
let ocProfiles = {};
|
|
let ocChart = null;
|
|
const OC_MAX_SAMPLES = 120;
|
|
|
|
function initOverclockChart() {
|
|
const canvas = document.getElementById('oc-chart');
|
|
if (!canvas || typeof Chart === 'undefined') return;
|
|
const ctx = canvas.getContext('2d');
|
|
|
|
ocChart = new Chart(ctx, {
|
|
type: 'line',
|
|
data: {
|
|
labels: [],
|
|
datasets: [
|
|
{ label: 'Core MHz', data: [], borderColor: '#22d3ee', backgroundColor: 'rgba(34,211,238,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: true },
|
|
{ label: 'Mem MHz', data: [], borderColor: '#c084fc', backgroundColor: 'rgba(192,132,252,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: false },
|
|
{ label: 'Temp °C', data: [], borderColor: '#fb7185', backgroundColor: 'rgba(251,113,133,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y1', fill: false },
|
|
{ label: 'Power W', data: [], borderColor: '#fbbf24', backgroundColor: 'rgba(251,191,36,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y2', fill: false },
|
|
{ label: 'Fan %', data: [], borderColor: '#34d399', backgroundColor: 'rgba(52,211,153,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y3', fill: false },
|
|
{ label: 'VRAM GB', data: [], borderColor: '#818cf8', backgroundColor: 'rgba(129,140,248,0.08)', borderWidth: 1.8, pointRadius: 0, tension: 0.35, yAxisID: 'y4', fill: false },
|
|
{ label: 'RAM Cache GB', data: [], borderColor: '#e879f9', backgroundColor: 'rgba(232,121,249,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y5', fill: false },
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
animation: false,
|
|
interaction: { mode: 'index', intersect: false },
|
|
scales: {
|
|
x: { ticks: { color: '#64748b', maxTicksLimit: 8, font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
|
|
y: { position: 'left', title: { display: true, text: 'MHz', color: '#22d3ee', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } },
|
|
y1: { position: 'right', title: { display: true, text: '°C', color: '#fb7185', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
|
|
y2: { position: 'right', offset: true, title: { display: true, text: 'W', color: '#fbbf24', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 400 },
|
|
y3: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 },
|
|
y4: { position: 'right', offset: true, title: { display: true, text: 'VRAM GB', color: '#818cf8', font: { size: 9 } }, ticks: { color: '#818cf8', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 16 },
|
|
y5: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 64 },
|
|
},
|
|
plugins: {
|
|
legend: { display: false },
|
|
tooltip: {
|
|
callbacks: {
|
|
label: function(context) {
|
|
const label = context.dataset.label || '';
|
|
const val = context.parsed.y;
|
|
if (label.includes('MHz')) return `${label}: ${val} MHz`;
|
|
if (label.includes('°C')) return `${label}: ${val}°C`;
|
|
if (label.includes('Power')) return `${label}: ${val} W`;
|
|
if (label.includes('Fan')) return `${label}: ${val}%`;
|
|
if (label.includes('GB')) return `${label}: ${val} GB`;
|
|
return `${label}: ${val}`;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
}
|
|
});
|
|
}
|
|
|
|
function pushOverclockSample(gpu, ram) {
|
|
if (!ocChart || !gpu || !gpu.available) return;
|
|
const label = new Date().toLocaleTimeString([], { hour12: false });
|
|
ocChart.data.labels.push(label);
|
|
ocChart.data.datasets[0].data.push(gpu.clock_graphics_mhz || 0);
|
|
ocChart.data.datasets[1].data.push(gpu.clock_mem_mhz || 0);
|
|
ocChart.data.datasets[2].data.push(gpu.temperature_c || 0);
|
|
ocChart.data.datasets[3].data.push(gpu.power_w || 0);
|
|
ocChart.data.datasets[4].data.push(gpu.fan_pct || 0);
|
|
ocChart.data.datasets[5].data.push(parseFloat(gpu.vram_used_gb) || 0);
|
|
ocChart.data.datasets[6].data.push(parseFloat(ram?.cached_gb) || 0);
|
|
if (ocChart.data.labels.length > OC_MAX_SAMPLES) {
|
|
ocChart.data.labels.shift();
|
|
ocChart.data.datasets.forEach(d => d.data.shift());
|
|
}
|
|
ocChart.update('none');
|
|
}
|
|
|
|
let initialOcLoaded = false;
|
|
|
|
async function fetchOverclockStatus() {
|
|
try {
|
|
const resp = await fetch('/api/overclock');
|
|
if (!resp.ok) return;
|
|
const data = await resp.json();
|
|
ocProfiles = data.profiles || {};
|
|
renderOverclockStatus(data);
|
|
if (!initialOcLoaded && data.active_profile) {
|
|
initialOcLoaded = true;
|
|
const editSel = document.getElementById('oc-edit-profile');
|
|
if (editSel) {
|
|
editSel.value = data.active_profile;
|
|
loadOverclockForProfile(data.active_profile);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.warn('Overclock fetch error:', err);
|
|
}
|
|
}
|
|
|
|
function renderOverclockStatus(data) {
|
|
const active = data.active_profile || 'balanced';
|
|
|
|
const badge = document.getElementById('oc-active-badge');
|
|
badge.textContent = `Active: ${active}`;
|
|
if (active === 'ollama') {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-purple-950 border border-purple-700 text-purple-300';
|
|
} else if (active === 'comfy') {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-cyan-950 border border-cyan-700 text-cyan-300';
|
|
} else {
|
|
badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-300';
|
|
}
|
|
|
|
const gpu = data.gpu || {};
|
|
document.getElementById('oc-power-limit').textContent = `${gpu.power_limit_w ?? '--'} W`;
|
|
document.getElementById('oc-core').textContent = `${gpu.clock_sm_mhz ?? '--'} MHz`;
|
|
document.getElementById('oc-mem').textContent = `${gpu.clock_mem_mhz ?? '--'} MHz`;
|
|
document.getElementById('oc-temp-draw').textContent = `${gpu.temp_c ?? '--'}°C / ${gpu.power_draw_w ?? '--'}W`;
|
|
|
|
// Specs strip
|
|
document.getElementById('oc-spec-gpu').textContent = gpu.name || '--';
|
|
document.getElementById('oc-spec-driver').textContent = gpu.driver_version || '--';
|
|
const vramGb = (gpu.vram_total_mb || 0) / 1024;
|
|
document.getElementById('oc-spec-vram').textContent = vramGb > 0 ? vramGb.toFixed(0) + ' GB' : '--';
|
|
document.getElementById('oc-spec-maxcore').textContent = (gpu.clock_sm_max_mhz ?? '--') + ' MHz';
|
|
document.getElementById('oc-spec-maxmem').textContent = (gpu.clock_mem_max_mhz ?? '--') + ' MHz';
|
|
document.getElementById('oc-spec-power').textContent = `${gpu.power_limit_w ?? '--'} / ${gpu.power_max_w ?? '--'} W`;
|
|
|
|
// Fan status
|
|
const fan = data.fan || {};
|
|
const fanBadge = document.getElementById('oc-fan-badge');
|
|
if (fanBadge) {
|
|
if (fan.manual) {
|
|
fanBadge.textContent = `MANUAL (${fan.target_speed_pct ?? '--'}%)`;
|
|
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-500 text-emerald-300 font-bold';
|
|
} else {
|
|
fanBadge.textContent = 'AUTO (VBIOS)';
|
|
fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-400 font-bold';
|
|
}
|
|
}
|
|
|
|
const xBadge = document.getElementById('oc-x-badge');
|
|
if (data.headless_x_running) {
|
|
xBadge.textContent = 'X: ON';
|
|
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-700 text-emerald-300';
|
|
} else {
|
|
xBadge.textContent = 'X: OFF';
|
|
xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-rose-950 border border-rose-700 text-rose-300';
|
|
}
|
|
|
|
const offsets = (data.last_result && data.last_result.offsets) || {};
|
|
const note = document.getElementById('oc-offset-note');
|
|
if (offsets.supported) {
|
|
note.textContent = '✅ Clock offsets active (proprietary driver)';
|
|
note.className = 'text-[11px] font-mono text-emerald-400';
|
|
} else {
|
|
note.textContent = '⚠️ Clock offsets off — nvidia-open lacks offset support (power + clock locks active)';
|
|
note.className = 'text-[11px] font-mono text-amber-400';
|
|
}
|
|
|
|
// highlight active profile button
|
|
['ollama', 'comfy', 'balanced'].forEach(p => {
|
|
const btn = document.getElementById('oc-btn-' + p);
|
|
if (p === active) {
|
|
btn.classList.add('ring-2', 'ring-fuchsia-400');
|
|
} else {
|
|
btn.classList.remove('ring-2', 'ring-fuchsia-400');
|
|
}
|
|
});
|
|
}
|
|
|
|
function loadOverclockForProfile(name) {
|
|
const p = ocProfiles[name];
|
|
if (!p) return;
|
|
document.getElementById('oc-slider-power').value = p.power_limit_w || 370;
|
|
document.getElementById('oc-val-power').textContent = (p.power_limit_w || 370) + ' W';
|
|
document.getElementById('oc-slider-core').value = p.core_offset_mhz || 0;
|
|
document.getElementById('oc-val-core').textContent = '+' + (p.core_offset_mhz || 0) + ' MHz';
|
|
document.getElementById('oc-slider-mem').value = p.mem_offset_mhz || 0;
|
|
document.getElementById('oc-val-mem').textContent = '+' + (p.mem_offset_mhz || 0) + ' MHz';
|
|
document.getElementById('oc-lockcore-toggle').checked = (p.lock_core_max || 0) > 0;
|
|
document.getElementById('oc-val-lockcore').textContent = (p.lock_core_max || 0) > 0 ? 'On' : 'Off';
|
|
document.getElementById('oc-lockmem-toggle').checked = (p.lock_mem_mhz || 0) > 0;
|
|
document.getElementById('oc-val-lockmem').textContent = (p.lock_mem_mhz || 0) > 0 ? 'On' : 'Off';
|
|
|
|
const fanMode = p.fan_mode || 'auto';
|
|
const profFanMode = document.getElementById('oc-prof-fanmode');
|
|
if (profFanMode) profFanMode.value = fanMode;
|
|
const profFanSpeed = document.getElementById('oc-slider-prof-fanspeed');
|
|
if (profFanSpeed) profFanSpeed.value = p.fan_speed_pct || 70;
|
|
const profFanVal = document.getElementById('oc-val-prof-fanspeed');
|
|
if (profFanVal) profFanVal.textContent = (p.fan_speed_pct || 70) + '%';
|
|
toggleProfileFanMode();
|
|
}
|
|
|
|
function toggleProfileFanMode() {
|
|
const mode = document.getElementById('oc-prof-fanmode').value;
|
|
const container = document.getElementById('oc-prof-fanspeed-container');
|
|
const valBadge = document.getElementById('oc-val-prof-fanmode');
|
|
if (valBadge) valBadge.textContent = (mode === 'manual') ? 'Manual Target' : 'Auto (VBIOS)';
|
|
if (container) {
|
|
if (mode === 'manual') {
|
|
container.classList.remove('opacity-50', 'pointer-events-none');
|
|
} else {
|
|
container.classList.add('opacity-50', 'pointer-events-none');
|
|
}
|
|
}
|
|
}
|
|
|
|
function toggleCoreLock() {
|
|
document.getElementById('oc-val-lockcore').textContent = document.getElementById('oc-lockcore-toggle').checked ? 'On' : 'Off';
|
|
}
|
|
|
|
function toggleMemLock() {
|
|
document.getElementById('oc-val-lockmem').textContent = document.getElementById('oc-lockmem-toggle').checked ? 'On' : 'Off';
|
|
}
|
|
|
|
async function setFanManual(speed) {
|
|
let pct = speed;
|
|
if (pct === undefined || pct === null) {
|
|
pct = parseInt(document.getElementById('oc-fan-slider').value);
|
|
} else {
|
|
pct = parseInt(pct);
|
|
const slider = document.getElementById('oc-fan-slider');
|
|
if (slider) slider.value = pct;
|
|
const label = document.getElementById('oc-fan-slider-label');
|
|
if (label) label.textContent = pct + '%';
|
|
}
|
|
try {
|
|
const resp = await fetch('/api/overclock/fan', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ mode: 'manual', percent: pct })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) alert(`Fan set failed: ${result.detail || 'error'}`);
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error setting fan speed: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function setFanAuto() {
|
|
try {
|
|
const resp = await fetch('/api/overclock/fan', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ mode: 'auto' })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) alert(`Fan auto failed: ${result.detail || 'error'}`);
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error setting fan to auto: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function applyOverclock(profile) {
|
|
try {
|
|
const resp = await fetch('/api/overclock/apply', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ profile })
|
|
});
|
|
const result = await resp.json();
|
|
if (!resp.ok) {
|
|
alert(`Apply failed: ${result.detail || 'error'}`);
|
|
}
|
|
const editSel = document.getElementById('oc-edit-profile');
|
|
if (editSel) {
|
|
editSel.value = profile;
|
|
loadOverclockForProfile(profile);
|
|
}
|
|
await fetchOverclockStatus();
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
async function saveOverclockProfile() {
|
|
const name = document.getElementById('oc-edit-profile').value;
|
|
const power = parseInt(document.getElementById('oc-slider-power').value);
|
|
const core = parseInt(document.getElementById('oc-slider-core').value);
|
|
const mem = parseInt(document.getElementById('oc-slider-mem').value);
|
|
const lockCore = document.getElementById('oc-lockcore-toggle').checked;
|
|
const lockMem = document.getElementById('oc-lockmem-toggle').checked;
|
|
const fanMode = document.getElementById('oc-prof-fanmode') ? document.getElementById('oc-prof-fanmode').value : 'auto';
|
|
const fanSpeed = document.getElementById('oc-slider-prof-fanspeed') ? parseInt(document.getElementById('oc-slider-prof-fanspeed').value) : 70;
|
|
|
|
const config = {
|
|
power_limit_w: power,
|
|
core_offset_mhz: core,
|
|
mem_offset_mhz: mem,
|
|
lock_core_min: lockCore ? 2900 : 0,
|
|
lock_core_max: lockCore ? 3105 : 0,
|
|
lock_mem_mhz: lockMem ? 11501 : 0,
|
|
fan_mode: fanMode,
|
|
fan_speed_pct: (fanMode === 'manual') ? fanSpeed : 0,
|
|
};
|
|
|
|
try {
|
|
const resp = await fetch(`/api/overclock/profiles/${name}`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ config })
|
|
});
|
|
const result = await resp.json();
|
|
if (resp.ok) {
|
|
await fetchOverclockStatus();
|
|
await applyOverclock(name);
|
|
} else {
|
|
alert(`Save failed: ${result.detail || 'error'}`);
|
|
}
|
|
} catch (err) {
|
|
alert(`Error: ${err}`);
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// THERMAL GOVERNOR / RESIDENCY / ANALYTICS / AUTOTUNE
|
|
// ============================================================================
|
|
|
|
let governorEnabled = true;
|
|
|
|
function renderGovernor(gov) {
|
|
if (!gov) return;
|
|
governorEnabled = gov.enabled;
|
|
const levels = 3;
|
|
const el = (id) => document.getElementById(id);
|
|
if (!el('gov-label')) return;
|
|
|
|
el('gov-label').textContent = gov.label || '—';
|
|
el('gov-label').className = 'text-2xl font-bold ' +
|
|
(gov.level === 0 ? 'text-emerald-400' : gov.level < 3 ? 'text-amber-400' : 'text-rose-400');
|
|
el('gov-level').textContent = `level ${gov.level} / ${levels}`;
|
|
el('gov-bar').style.width = `${(gov.level / levels) * 100}%`;
|
|
el('gov-esc').textContent = `${gov.escalate_at_c}°C`;
|
|
el('gov-rec').textContent = `${gov.recover_below_c}°C`;
|
|
el('gov-scale').textContent = `${Math.round((gov.offset_scale ?? 1) * 100)}%`;
|
|
el('gov-reason').textContent = gov.last_reason || '—';
|
|
|
|
const toggle = el('gov-toggle');
|
|
toggle.textContent = gov.enabled ? 'Enabled' : 'Disabled';
|
|
toggle.className = 'px-2.5 py-1 text-xs font-semibold rounded-lg border transition ' +
|
|
(gov.enabled ? 'bg-emerald-950/70 border-emerald-800 text-emerald-300 hover:bg-emerald-900'
|
|
: 'bg-slate-800 border-slate-700 text-slate-400 hover:bg-slate-700');
|
|
|
|
const hist = el('gov-history');
|
|
hist.innerHTML = (gov.history || []).map(h => {
|
|
const t = new Date(h.ts * 1000).toLocaleTimeString();
|
|
const up = h.to_level > h.from_level;
|
|
return `<div class="flex justify-between ${up ? 'text-rose-400' : 'text-cyan-400'}">
|
|
<span>${t} ${up ? '▲' : '▼'} L${h.to_level} ${h.label}</span>
|
|
<span class="text-slate-500 truncate ml-2">${h.reason || ''}</span></div>`;
|
|
}).join('') || '<div class="text-slate-600">No derates recorded — the card has stayed happy.</div>';
|
|
}
|
|
|
|
async function toggleGovernor() {
|
|
const res = await fetch('/api/governor', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ enabled: !governorEnabled })
|
|
});
|
|
renderGovernor(await res.json());
|
|
}
|
|
|
|
async function resetGovernor() {
|
|
const res = await fetch('/api/governor', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ reset: true })
|
|
});
|
|
renderGovernor(await res.json());
|
|
}
|
|
|
|
// ---------------------------------------------------------------- residency
|
|
|
|
async function fetchCacheReport(refresh = false) {
|
|
const body = document.getElementById('cache-files');
|
|
if (!body) return;
|
|
if (refresh) body.innerHTML = '<div class="text-slate-500 text-xs">Scanning…</div>';
|
|
try {
|
|
const res = await fetch(`/api/cache/report?refresh=${refresh ? 'true' : 'false'}`);
|
|
const d = await res.json();
|
|
|
|
document.getElementById('cache-total').textContent =
|
|
`${d.resident_total_gb} / ${d.catalog_total_gb} GB`;
|
|
document.getElementById('cache-meta').textContent =
|
|
`${d.residency_pct}% resident · ${d.warm_files}/${d.files_scanned} warm · ${d.scan_ms}ms`;
|
|
|
|
const cap = d.capability || {};
|
|
document.getElementById('cache-capability').innerHTML = cap.exact_everywhere
|
|
? '<span class="text-emerald-500">Exact measurement available for every model file.</span>'
|
|
: `<span class="text-amber-500">Ollama weights measured by read-rate probe</span> — ${cap.reason || ''}`;
|
|
|
|
body.innerHTML = (d.files || []).slice(0, 40).map(f => {
|
|
const pct = f.resident_pct ?? 0;
|
|
const color = pct >= 90 ? 'bg-emerald-500' : pct >= 25 ? 'bg-amber-500' : 'bg-slate-700';
|
|
const kind = f.kind === 'ollama' ? 'text-cyan-400' : 'text-fuchsia-400';
|
|
return `<div>
|
|
<div class="flex justify-between text-[11px] font-mono">
|
|
<span class="${kind} truncate mr-2" title="${f.full_path}">${f.name}</span>
|
|
<span class="text-slate-400 whitespace-nowrap">${pct}% · ${f.size_gb}GB</span>
|
|
</div>
|
|
<div class="w-full bg-slate-950 rounded-full h-1 mt-0.5 overflow-hidden">
|
|
<div class="${color} h-full transition-all" style="width:${pct}%"></div>
|
|
</div>
|
|
</div>`;
|
|
}).join('');
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400 text-xs">Residency scan failed: ${e}</div>`;
|
|
}
|
|
}
|
|
|
|
async function fetchWarmPlan() {
|
|
const body = document.getElementById('cache-files');
|
|
body.innerHTML = '<div class="text-slate-500 text-xs">Building plan…</div>';
|
|
try {
|
|
const d = await (await fetch('/api/warm-plan')).json();
|
|
document.getElementById('cache-total').textContent = `${d.planned_gb} GB planned`;
|
|
document.getElementById('cache-meta').textContent =
|
|
`budget ${d.budget_gb} GB · ${d.warm_count} to warm · ${d.skipped_count} skipped`;
|
|
document.getElementById('cache-capability').innerHTML =
|
|
'<span class="text-slate-500">Ranked by recency/frequency; already-resident files cost nothing.</span>';
|
|
body.innerHTML = d.plan.map((p, i) => `<div class="flex justify-between text-[11px] font-mono">
|
|
<span class="text-amber-400 truncate mr-2">${i + 1}. ${p.name}</span>
|
|
<span class="text-slate-400 whitespace-nowrap">${p.size_gb}GB · ${p.resident_pct}% · s=${p.score}</span>
|
|
</div>`).join('') || '<div class="text-emerald-500 text-xs">Everything worth warming is already resident.</div>';
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400 text-xs">${e}</div>`;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- analytics
|
|
|
|
async function fetchAnalytics() {
|
|
const el = document.getElementById('analytics-body');
|
|
if (!el) return;
|
|
const days = document.getElementById('analytics-days').value;
|
|
try {
|
|
const d = await (await fetch(`/api/analytics/profiles?days=${days}`)).json();
|
|
if (!d.profiles || !d.profiles.length) {
|
|
el.innerHTML = '<div class="text-slate-500">No swaps recorded yet in this window. Switch a model to start building the comparison.</div>';
|
|
return;
|
|
}
|
|
const best = Math.max(...d.profiles.map(p => p.avg_tok_s || 0));
|
|
el.innerHTML = `<table class="w-full text-[11px] font-mono">
|
|
<thead><tr class="text-slate-500 text-left border-b border-slate-800">
|
|
<th class="pb-1">profile</th><th>tok/s</th><th>load GB/s</th><th>°C avg</th><th>W avg</th><th>SM MHz</th><th>n</th>
|
|
</tr></thead><tbody>` +
|
|
d.profiles.map(p => {
|
|
const win = (p.avg_tok_s || 0) === best;
|
|
return `<tr class="border-b border-slate-900 ${win ? 'text-emerald-400' : 'text-slate-300'}">
|
|
<td class="py-1">${win ? '★ ' : ''}${p.profile ?? '—'}</td>
|
|
<td>${p.avg_tok_s ?? '—'}</td><td>${p.avg_load_gbps ?? '—'}</td>
|
|
<td>${p.avg_temp_c ?? '—'}</td><td>${p.avg_power_w ?? '—'}</td>
|
|
<td>${p.avg_clock_sm ?? '—'}</td><td class="text-slate-500">${p.swaps}</td></tr>`;
|
|
}).join('') + '</tbody></table>';
|
|
} catch (e) {
|
|
el.innerHTML = `<div class="text-rose-400">${e}</div>`;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- autotune
|
|
|
|
let sweepPoll = null;
|
|
|
|
async function runSweep() {
|
|
const btn = document.getElementById('at-run');
|
|
const body = document.getElementById('autotune-body');
|
|
const payload = {
|
|
knob: document.getElementById('at-knob').value,
|
|
profile: document.getElementById('at-profile').value,
|
|
start: parseInt(document.getElementById('at-start').value, 10),
|
|
stop: parseInt(document.getElementById('at-stop').value, 10),
|
|
step: parseInt(document.getElementById('at-step').value, 10),
|
|
apply_best: document.getElementById('at-apply').checked,
|
|
};
|
|
btn.disabled = true;
|
|
btn.classList.add('opacity-50');
|
|
body.innerHTML = '<div class="text-fuchsia-400">Sweeping… each step loads the model and benchmarks decode. This takes a few minutes.</div>';
|
|
sweepPoll = setInterval(pollSweep, 2000);
|
|
try {
|
|
const res = await fetch('/api/autotune/sweep', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload)
|
|
});
|
|
const d = await res.json();
|
|
renderSweep(res.ok ? d : { error: d.detail });
|
|
} catch (e) {
|
|
body.innerHTML = `<div class="text-rose-400">${e}</div>`;
|
|
} finally {
|
|
clearInterval(sweepPoll);
|
|
btn.disabled = false;
|
|
btn.classList.remove('opacity-50');
|
|
document.getElementById('autotune-state').textContent = 'idle';
|
|
}
|
|
}
|
|
|
|
async function pollSweep() {
|
|
try {
|
|
const d = await (await fetch('/api/autotune')).json();
|
|
const s = document.getElementById('autotune-state');
|
|
if (d.running && d.current) {
|
|
s.textContent = `${d.current.knob}=${d.current.value} → ${d.current.tokens_per_sec} tok/s`;
|
|
s.className = 'text-xs font-mono text-fuchsia-400';
|
|
}
|
|
} catch (e) { /* transient */ }
|
|
}
|
|
|
|
async function cancelSweep() {
|
|
await fetch('/api/autotune/cancel', { method: 'POST' });
|
|
document.getElementById('autotune-state').textContent = 'cancelling…';
|
|
}
|
|
|
|
function renderSweep(d) {
|
|
const body = document.getElementById('autotune-body');
|
|
if (d.error) { body.innerHTML = `<div class="text-rose-400">${d.error}</div>`; return; }
|
|
const rows = (d.table || []).map(r => `<tr class="${r.stable ? 'text-slate-300' : 'text-rose-400'} border-b border-slate-900">
|
|
<td class="py-0.5">${r.value}</td><td>${r.tokens_per_sec}</td><td>${r.temp_c}</td>
|
|
<td>${r.power_w ?? '—'}</td><td>${r.clock_mem_mhz ?? '—'}</td>
|
|
<td class="text-slate-500 truncate">${r.instability || 'stable'}</td></tr>`).join('');
|
|
const best = d.best
|
|
? `<div class="text-emerald-400 mb-2">★ Best stable: <span class="font-mono">${d.knob}=${d.best.value}</span>
|
|
→ ${d.best.tokens_per_sec} tok/s at ${d.best.temp_c}°C
|
|
${d.gain_vs_current_pct != null
|
|
? `<span class="text-slate-400">(${d.gain_vs_current_pct > 0 ? '+' : ''}${d.gain_vs_current_pct}% vs your current setting)</span>`
|
|
: d.gain_vs_first_step_pct != null
|
|
? `<span class="text-slate-500">(spread across tried values: ${d.gain_vs_first_step_pct > 0 ? '+' : ''}${d.gain_vs_first_step_pct}% vs slowest)</span>`
|
|
: ''}
|
|
${d.applied_to_profile ? '<span class="text-fuchsia-400">· saved to profile</span>' : ''}</div>`
|
|
: '<div class="text-amber-400 mb-2">No stable step produced throughput.</div>';
|
|
const unstable = d.first_unstable
|
|
? `<div class="text-rose-400 text-[11px] mb-2">Stopped climbing at ${d.first_unstable.value}: ${d.first_unstable.why}</div>` : '';
|
|
body.innerHTML = best + unstable + `<table class="w-full text-[11px] font-mono">
|
|
<thead><tr class="text-slate-500 text-left border-b border-slate-800">
|
|
<th class="pb-1">offset</th><th>tok/s</th><th>°C</th><th>W</th><th>mem MHz</th><th>status</th>
|
|
</tr></thead><tbody>${rows}</tbody></table>`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- wiring
|
|
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
fetchCacheReport(false);
|
|
fetchAnalytics();
|
|
setInterval(fetchAnalytics, 60000);
|
|
setInterval(() => fetchCacheReport(false), 30000);
|
|
fetch('/api/autotune').then(r => r.json()).then(d => {
|
|
if (d.last_result) renderSweep(d.last_result);
|
|
}).catch(() => {});
|
|
});
|
|
|
|
|
|
// ---------------------------------------------------------------- arbitration
|
|
|
|
function renderArbitrator(arb, gpu) {
|
|
const el = (id) => document.getElementById(id);
|
|
if (!el('arb-action')) return;
|
|
const c = arb.counters || {};
|
|
|
|
// VRAM held by processes HyperSwap cannot reclaim. Worth showing: it is headroom the
|
|
// arbitrator can never give back, no matter how much it purges.
|
|
const bd = (gpu && gpu.breakdown) || {};
|
|
const un = el('arb-unmanaged');
|
|
if (un) {
|
|
const procs = bd.unmanaged || [];
|
|
un.innerHTML = procs.length
|
|
? `<span class="text-amber-400">${bd.unmanaged_gb} GB unreclaimable</span> — ` +
|
|
procs.map(p => `${p.name} (${p.vram_mb} MB)`).join(', ')
|
|
: '';
|
|
}
|
|
|
|
el('arb-action').textContent = arb.last_action || 'Idle';
|
|
el('arb-yields').textContent = c.yields ?? 0;
|
|
el('arb-busy').textContent = c.yield_deferred_busy ?? 0;
|
|
el('arb-later').textContent = c.deferred_releases ?? 0;
|
|
el('arb-stalled').textContent = c.yield_stalled ?? 0;
|
|
el('arb-purges').textContent = c.purges ?? 0;
|
|
el('arb-defpurge').textContent = c.deferred_purges ?? 0;
|
|
|
|
const ws = el('arb-ws');
|
|
ws.textContent = arb.connected_ws ? 'ComfyUI WS live' : 'WS down — polling';
|
|
ws.className = 'text-xs font-mono ' + (arb.connected_ws ? 'text-emerald-400' : 'text-amber-400');
|
|
|
|
// Show why we are holding off, and the idle countdown before ComfyUI is purged.
|
|
const parts = [];
|
|
const backoff = arb.yield_backoff || {};
|
|
for (const [model, secs] of Object.entries(backoff)) {
|
|
parts.push(`waiting ${secs}s before asking '${model}' again`);
|
|
}
|
|
if (arb.pending_purge && arb.comfy_idle_s != null) {
|
|
const left = Math.max((arb.idle_purge_after_s || 0) - arb.comfy_idle_s, 0).toFixed(0);
|
|
parts.push(`ComfyUI idle ${arb.comfy_idle_s}s — holding its checkpoints ${left}s longer`);
|
|
}
|
|
el('arb-backoff').textContent = parts.join(' · ');
|
|
}
|
|
|
|
|
|
// ---------------------------------------------------------------- health
|
|
|
|
async function fetchHealth(verbose = false) {
|
|
const badge = document.getElementById('health-badge');
|
|
const body = document.getElementById('health-body');
|
|
if (!badge) return;
|
|
try {
|
|
const d = await (await fetch('/api/health')).json();
|
|
const style = {
|
|
ok: 'bg-emerald-950/70 border border-emerald-800 text-emerald-300',
|
|
degraded: 'bg-amber-950/70 border border-amber-800 text-amber-300',
|
|
failed: 'bg-rose-950/70 border border-rose-800 text-rose-300',
|
|
}[d.status];
|
|
badge.className = `px-2 py-1 text-xs font-bold rounded-lg ${style}`;
|
|
badge.textContent = `${d.status.toUpperCase()} · ${d.summary}`;
|
|
|
|
// Healthy checks collapse to one line; anything wrong is shown in full with the
|
|
// impact and the fix, because that is the information you actually need.
|
|
const problems = d.checks.filter(c => c.status !== 'ok');
|
|
const shown = (verbose || problems.length) ? (verbose ? d.checks : problems) : [];
|
|
body.innerHTML = shown.map(c => {
|
|
const colour = c.status === 'ok' ? 'text-emerald-400'
|
|
: c.status === 'degraded' ? 'text-amber-400' : 'text-rose-400';
|
|
const mark = c.status === 'ok' ? '✓' : c.status === 'degraded' ? '!' : '✗';
|
|
let html = `<div><span class="${colour}">${mark} ${c.name}</span>` +
|
|
`<span class="text-slate-400"> — ${c.detail}</span></div>`;
|
|
if (c.status !== 'ok') {
|
|
if (c.impact) html += `<div class="text-slate-500 pl-4">impact: ${c.impact}</div>`;
|
|
if (c.fix) html += `<div class="text-slate-500 pl-4">fix: ${c.fix}</div>`;
|
|
}
|
|
return html;
|
|
}).join('') || '<div class="text-emerald-500">All dependencies healthy.</div>';
|
|
} catch (e) {
|
|
badge.className = 'px-2 py-1 text-xs font-bold rounded-lg bg-rose-950/70 border border-rose-800 text-rose-300';
|
|
badge.textContent = 'UNREACHABLE';
|
|
body.innerHTML = `<div class="text-rose-400">${e}</div>`;
|
|
}
|
|
}
|
|
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
fetchHealth(false);
|
|
setInterval(() => fetchHealth(false), 60000);
|
|
});
|