From 1f198ae10ebc14c297469638044244c7f606444c Mon Sep 17 00:00:00 2001 From: drjones Date: Sun, 23 Aug 2026 08:56:06 -0700 Subject: [PATCH] Add GPU fan control, live telemetry, and per-profile fan curves in HyperSwap dashboard --- README.md | 18 ++- mcp_server.py | 16 ++ overclock_manager.py | 339 ++++++++++++++++++++++++++++++++++++++++ overclock_profiles.json | 35 +++++ server.py | 68 ++++++++ static/app.js | 323 ++++++++++++++++++++++++++++++++++++++ static/index.html | 248 ++++++++++++++++++++++++++++- static/styles.css | 10 ++ vram_arbitrator.py | 186 +++++++++++++++++++++- 9 files changed, 1237 insertions(+), 6 deletions(-) create mode 100644 overclock_manager.py create mode 100644 overclock_profiles.json diff --git a/README.md b/README.md index d187b98..43b3c91 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,19 @@ The HyperSwap server runs on port `9090` by default. Interactive OpenAPI/Swagger Returns a unified JSON snapshot of all system sensors, GPU processes, host RAM, Ollama status, ComfyUI queue, and transition logs. #### `GET /api/gpu` -Returns hardware sensors (utilization %, temperature, power draw in Watts, fan %, GPU graphics/memory clocks, and active PIDs). +Returns hardware sensors (utilization %, temperature, power draw in Watts, fan speed %, per-fan telemetry, GPU graphics/memory clocks, and active PIDs). + +#### `GET /api/overclock/fan` / `GET /api/gpu/fan` +Returns current GPU fan mode (`auto` vs `manual`), target fan speed %, and live fan telemetry. + +#### `POST /api/overclock/fan` / `POST /api/gpu/fan` +Sets GPU fan speed mode (`auto` or `manual`) with target speed % (30–100%). + +#### `GET /api/overclock` +Returns active overclock profile, configured profiles, GPU clock limits, and fan status. + +#### `POST /api/overclock/apply` +Applies a named profile (`ollama`, `comfy`, `balanced`) configuring power limits, clock locks, offsets, and fan speed. #### `GET /api/memory` Returns precise `/proc/meminfo` metrics including Total, Used, OS Page Cache, and free memory. @@ -121,7 +133,9 @@ HyperSwap includes a native **MCP 2.0 server** (`mcp_server.py`) that exposes al | Tool Name | Parameters | Description | | :--- | :--- | :--- | -| **`get_gpu_status`** | *None* | Live NVIDIA GPU hardware telemetry, VRAM breakdown, temps, power, and PIDs. | +| **`get_gpu_status`** | *None* | Live NVIDIA GPU hardware telemetry, VRAM breakdown, temps, power, fan %, and PIDs. | +| **`get_gpu_fan_status`** | *None* | Current GPU fan mode (`auto`/`manual`) and target fan percentage. | +| **`set_gpu_fan_speed`** | `mode` (str, "auto"\|"manual"), `percent` (optional int) | Sets fan speed mode and target PWM % (30–100%). | | **`get_host_memory_status`** | *None* | 64GB host RAM breakdown, active page cache size, and cache ratio. | | **`switch_ollama_model`** | `model_name` (str), `keep_alive` (str, default "30m") | Hot-swaps active LLM in VRAM, measures latency (ms) and tokens/sec. | | **`soft_yield_ollama_vram`** | `model_name` (optional str) | Yields Ollama VRAM to 0 MB in ~15ms while keeping model weights in RAM cache. | diff --git a/mcp_server.py b/mcp_server.py index d758e74..7851760 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -9,6 +9,7 @@ from typing import Dict, List, Any, Optional from mcp.server import MCPServer import ram_optimizer import vram_arbitrator +import overclock_manager logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") logger = logging.getLogger("gpu_swapper_mcp") @@ -117,6 +118,21 @@ async def run_model_switch_benchmark(iterations: int = 2) -> str: "rounds": results, }, indent=2) +@mcp.tool() +def get_gpu_fan_status() -> str: + """Get current GPU fan control mode (auto vs manual) and target speed.""" + status = overclock_manager.get_fan_status() + return json.dumps(status, indent=2) + +@mcp.tool() +def set_gpu_fan_speed(mode: str = "auto", percent: Optional[int] = None) -> str: + """Set GPU fan speed mode ('auto' or 'manual') with target percent (30-100%).""" + if mode.lower() == "manual" and percent is not None: + res = overclock_manager.set_fan_speed(percent) + else: + res = overclock_manager.set_fan_auto() + return json.dumps(res, indent=2) + # ========================================== # MCP RESOURCES # ========================================== diff --git a/overclock_manager.py b/overclock_manager.py new file mode 100644 index 0000000..60a6c4b --- /dev/null +++ b/overclock_manager.py @@ -0,0 +1,339 @@ +""" +Overclock Manager for HyperSwap — per-app GPU overclock profiles for the RTX 4080 SUPER. + +Lever hierarchy (what actually works on this box): + 1. Power limit nvidia-smi -pl -> 320W -> 370W max (BIG win, works on open module) + 2. Clock locks nvidia-smi -lgc / -lmc -> sustain max boost (works on open module) + 3. Clock offsets nvidia-settings -a ... -> +core / +mem beyond stock (needs PROPRIETARY module) + +Profiles are application-specific: + - ollama : LLM decode is memory-bandwidth bound -> lock memory clock to max + max power + - comfy : diffusion is compute bound -> lock core clock high + max power + - balanced: stock boost, power unlocked only + +Auto-switches in lockstep with the VRAM arbitrator (vram_arbitrator.AutoArbitrator). +""" +import json +import logging +import os +import shutil +import subprocess +from typing import Dict, Any, Optional, List + +logger = logging.getLogger("overclock_manager") + +_BASE = os.path.dirname(os.path.abspath(__file__)) +CONFIG_PATH = os.path.join(_BASE, "overclock_profiles.json") + +NVIDIA_SMI = "nvidia-smi" +NVIDIA_SETTINGS = "nvidia-settings" +HEADLESS_DISPLAY = ":8" # dedicated headless X server owning the NVIDIA GPU +HEADLESS_CONFIG = "/etc/X11/xorg.conf.nvidia-headless" + +# Default profile set. lock_* == 0 means "don't lock" (let boost manage). +DEFAULT_PROFILES: Dict[str, Dict[str, Any]] = { + "ollama": { + "label": "Ollama — LLM decode (memory-bandwidth bound)", + "power_limit_w": 370, + "core_offset_mhz": 100, + "mem_offset_mhz": 500, + "lock_core_min": 0, + "lock_core_max": 0, + "lock_mem_mhz": 11501, + }, + "comfy": { + "label": "ComfyUI — diffusion (core-compute bound)", + "power_limit_w": 370, + "core_offset_mhz": 100, + "mem_offset_mhz": 500, + "lock_core_min": 2900, + "lock_core_max": 3105, + "lock_mem_mhz": 0, + }, + "balanced": { + "label": "Balanced — stock boost, power unlocked", + "power_limit_w": 370, + "core_offset_mhz": 0, + "mem_offset_mhz": 0, + "lock_core_min": 0, + "lock_core_max": 0, + "lock_mem_mhz": 0, + }, +} + +ACTIVE_PROFILE = "balanced" +_LAST_RESULT: Dict[str, Any] = {} +FAN_MANUAL = False + + +def _sh(cmd: List[str], use_sudo: bool = True, timeout: int = 10) -> Dict[str, Any]: + """Run a command; return rc/stdout/stderr. Prefers passwordless sudo.""" + full = list(cmd) + if use_sudo: + full = ["sudo", "-n"] + full + try: + proc = subprocess.run( + full, capture_output=True, text=True, timeout=timeout + ) + return {"rc": proc.returncode, "out": proc.stdout.strip(), "err": proc.stderr.strip()} + except subprocess.TimeoutExpired: + return {"rc": -1, "out": "", "err": "timeout"} + except FileNotFoundError as e: + return {"rc": -1, "out": "", "err": f"not found: {e}"} + + +def _smi(*args: str) -> Dict[str, Any]: + return _sh([NVIDIA_SMI, *args], use_sudo=True) + + +def _nvidia_settings(*args: str) -> Dict[str, Any]: + """Run nvidia-settings against the headless X display that owns the GPU.""" + env = os.environ.copy() + env["DISPLAY"] = HEADLESS_DISPLAY + cmd = [NVIDIA_SETTINGS, "-c", HEADLESS_DISPLAY, *args] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=15, env=env) + return {"rc": proc.returncode, "out": proc.stdout.strip(), "err": proc.stderr.strip()} + except Exception as e: + return {"rc": -1, "out": "", "err": str(e)} + + +def load_profiles() -> Dict[str, Dict[str, Any]]: + """Load profiles from disk, falling back to defaults and merging new keys.""" + profiles = json.loads(json.dumps(DEFAULT_PROFILES)) + if os.path.exists(CONFIG_PATH): + try: + with open(CONFIG_PATH) as f: + stored = json.load(f) + for name, cfg in stored.items(): + if name in profiles: + profiles[name].update(cfg) + else: + profiles[name] = cfg + except Exception as e: + logger.warning(f"Could not load {CONFIG_PATH}: {e}") + return profiles + + +def save_profiles(profiles: Dict[str, Dict[str, Any]]) -> bool: + try: + with open(CONFIG_PATH, "w") as f: + json.dump(profiles, f, indent=2) + return True + except Exception as e: + logger.error(f"save_profiles failed: {e}") + return False + + +def get_profiles() -> Dict[str, Dict[str, Any]]: + return load_profiles() + + +def set_profile(name: str, cfg: Dict[str, Any]) -> Dict[str, Any]: + profiles = load_profiles() + if name not in profiles: + return {"success": False, "error": f"unknown profile '{name}'"} + profiles[name].update(cfg) + ok = save_profiles(profiles) + return {"success": ok, "profiles": profiles if ok else None} + + +def _apply_power_limit(watts: int) -> Dict[str, Any]: + r = _smi("-pl", str(watts)) + ok = r["rc"] == 0 + return {"applied": ok, "detail": r.get("out") or r.get("err")} + + +def _apply_clock_lock(core_min: int, core_max: int) -> Dict[str, Any]: + if core_min == 0 and core_max == 0: + r = _smi("-rgc") + return {"applied": r["rc"] == 0, "detail": "reset"} + r = _smi("-lgc", f"{core_min},{core_max}") + return {"applied": r["rc"] == 0, "detail": r.get("out") or r.get("err")} + + +def _apply_mem_lock(mem_mhz: int) -> Dict[str, Any]: + if mem_mhz == 0: + r = _smi("-rmc") + return {"applied": r["rc"] == 0, "detail": "reset"} + r = _smi("-lmc", str(mem_mhz)) + return {"applied": r["rc"] == 0, "detail": r.get("out") or r.get("err")} + + +def _apply_offsets(core_mhz: int, mem_mhz: int) -> Dict[str, Any]: + """Apply +core/+mem offsets via nvidia-settings. Returns whether they actually stuck. + NOTE: offsets only work with the PROPRIETARY kernel module, not nvidia-open.""" + if core_mhz == 0 and mem_mhz == 0: + r = _nvidia_settings("-a", "[gpu:0]/GPUGraphicsClockOffset[3]=0", + "-a", "[gpu:0]/GPUMemoryTransferRateOffset[3]=0") + return {"applied": r["rc"] == 0, "detail": "reset", "supported": True} + + r = _nvidia_settings("-a", f"[gpu:0]/GPUGraphicsClockOffset[3]={core_mhz}", + "-a", f"[gpu:0]/GPUMemoryTransferRateOffset[3]={mem_mhz}") + if r["rc"] != 0: + return {"applied": False, "detail": r.get("err") or r.get("out"), "supported": False} + + # Read back to confirm the driver actually persisted the offsets. + q = _nvidia_settings("-q", "[gpu:0]/GPUGraphicsClockOffset[3]", + "-q", "[gpu:0]/GPUMemoryTransferRateOffset[3]") + applied_core = applied_mem = None + for line in q["out"].splitlines(): + line = line.strip() + if "GPUGraphicsClockOffset" in line and ":" in line and "(" in line: + try: + applied_core = int(line.split("):")[-1].split(".")[0].strip()) + except Exception: + pass + if "GPUMemoryTransferRateOffset" in line and ":" in line and "(" in line: + try: + applied_mem = int(line.split("):")[-1].split(".")[0].strip()) + except Exception: + pass + + supported = (applied_core is not None and applied_core != 0) or \ + (applied_mem is not None and applied_mem != 0) + return { + "applied": supported, + "supported": supported, + "readback_core": applied_core, + "readback_mem": applied_mem, + "detail": f"core readback={applied_core}, mem readback={applied_mem}", + } + + +def apply_profile(name: str) -> Dict[str, Any]: + """Apply a named overclock profile to the GPU. Returns a full result report.""" + global ACTIVE_PROFILE, _LAST_RESULT + profiles = load_profiles() + if name not in profiles: + return {"success": False, "error": f"unknown profile '{name}'", "profile": name} + + cfg = profiles[name] + fan_mode = cfg.get("fan_mode", "auto") + fan_speed = int(cfg.get("fan_speed_pct", 0)) + + result = { + "success": True, + "profile": name, + "label": cfg.get("label", name), + "power_limit": _apply_power_limit(int(cfg.get("power_limit_w", 370))), + "clock_lock": _apply_clock_lock(int(cfg.get("lock_core_min", 0)), int(cfg.get("lock_core_max", 0))), + "mem_lock": _apply_mem_lock(int(cfg.get("lock_mem_mhz", 0))), + "offsets": _apply_offsets(int(cfg.get("core_offset_mhz", 0)), int(cfg.get("mem_offset_mhz", 0))), + "fan": apply_fan_control(fan_mode, fan_speed), + } + result["gpu"] = get_gpu_state() + result["fan_status"] = get_fan_status() + + ACTIVE_PROFILE = name + _LAST_RESULT = result + logger.info(f"Overclock profile applied: {name} -> {json.dumps(result, default=str)}") + return result + + +def get_gpu_state() -> Dict[str, Any]: + """Read back live GPU clocks/power/limits via nvidia-smi.""" + state: Dict[str, Any] = {} + r = _smi( + "--query-gpu=driver_version,name,memory.total,power.limit,power.max_limit,power.default_limit," + "clocks.sm,clocks.max.sm,clocks.mem,clocks.max.mem," + "temperature.gpu,power.draw,fan.speed", + "--format=csv,noheader,nounits", + ) + if r["rc"] == 0 and r["out"]: + parts = [p.strip() for p in r["out"].split(",")] + keys = ["driver_version", "name", "vram_total_mb", "power_limit_w", "power_max_w", "power_default_w", + "clock_sm_mhz", "clock_sm_max_mhz", "clock_mem_mhz", "clock_mem_max_mhz", + "temp_c", "power_draw_w", "fan_pct"] + for i, k in enumerate(keys): + if i < len(parts): + try: + state[k] = float(parts[i]) + except ValueError: + state[k] = parts[i] + return state + + +def is_headless_x_running() -> bool: + r = _sh(["pgrep", "-f", f"Xorg {HEADLESS_DISPLAY}"], use_sudo=False) + return r["rc"] == 0 + + +def apply_fan_control(mode: str, speed_pct: int) -> Dict[str, Any]: + global FAN_MANUAL + if mode == "auto": + r = _nvidia_settings("-a", "[gpu:0]/GPUFanControlState=0") + ok = r["rc"] == 0 + if ok: + FAN_MANUAL = False + return {"applied": ok, "mode": "auto", "detail": r.get("out") or r.get("err")} + + speed_pct = max(30, min(100, int(speed_pct))) + r = _nvidia_settings( + "-a", "[gpu:0]/GPUFanControlState=1", + "-a", f"[fan:0]/GPUTargetFanSpeed={speed_pct}", + "-a", f"[fan:1]/GPUTargetFanSpeed={speed_pct}" + ) + ok = (r["rc"] == 0) + if ok: + FAN_MANUAL = True + return {"applied": ok, "mode": "manual", "speed_pct": speed_pct, "detail": r.get("out") or r.get("err")} + + +def set_fan_speed(percent: int) -> Dict[str, Any]: + """Set manual GPU fan target speed (30-100%).""" + return apply_fan_control(mode="manual", speed_pct=percent) + + +def set_fan_auto() -> Dict[str, Any]: + """Return GPU fan to automatic control.""" + global FAN_MANUAL + r = _nvidia_settings("-a", "[gpu:0]/GPUFanControlState=0") + ok = r["rc"] == 0 + if ok: + FAN_MANUAL = False + return {"success": ok, "manual": False, "fan_speed_pct": None, "detail": r.get("out") or r.get("err")} + + +def get_fan_status() -> Dict[str, Any]: + """Read current fan control mode + target speed.""" + global FAN_MANUAL + target = None + manual = FAN_MANUAL + r = _nvidia_settings("-q", "[gpu:0]/GPUFanControlState", "-q", "[fan:0]/GPUTargetFanSpeed") + if r.get("rc") == 0 and r.get("out"): + for line in r["out"].splitlines(): + if "GPUFanControlState" in line and ":" in line: + try: + val = int(line.split("):")[-1].split(".")[0].strip()) + manual = (val == 1) + except Exception: + pass + elif "GPUTargetFanSpeed" in line and ":" in line and "(" in line: + try: + target = int(line.split("):")[-1].split(".")[0].strip()) + except Exception: + pass + return {"manual": manual, "mode": "manual" if manual else "auto", "target_speed_pct": target} + + +def get_status() -> Dict[str, Any]: + """Full overclock status for the dashboard.""" + return { + "active_profile": ACTIVE_PROFILE, + "profiles": load_profiles(), + "gpu": get_gpu_state(), + "fan": get_fan_status(), + "headless_x_running": is_headless_x_running(), + "headless_display": HEADLESS_DISPLAY, + "last_result": _LAST_RESULT, + } + + +if __name__ == "__main__": + import sys + logging.basicConfig(level=logging.INFO) + if len(sys.argv) > 1: + print(json.dumps(apply_profile(sys.argv[1]), indent=2, default=str)) + else: + print(json.dumps(get_status(), indent=2, default=str)) diff --git a/overclock_profiles.json b/overclock_profiles.json new file mode 100644 index 0000000..eac830f --- /dev/null +++ b/overclock_profiles.json @@ -0,0 +1,35 @@ +{ + "ollama": { + "label": "Ollama \u2014 LLM decode (memory-bandwidth bound)", + "power_limit_w": 370, + "core_offset_mhz": 100, + "mem_offset_mhz": 700, + "lock_core_min": 0, + "lock_core_max": 0, + "lock_mem_mhz": 0, + "fan_mode": "auto", + "fan_speed_pct": 0 + }, + "comfy": { + "label": "ComfyUI \u2014 diffusion (core-compute bound)", + "power_limit_w": 370, + "core_offset_mhz": 100, + "mem_offset_mhz": 500, + "lock_core_min": 2900, + "lock_core_max": 3105, + "lock_mem_mhz": 0, + "fan_mode": "auto", + "fan_speed_pct": 0 + }, + "balanced": { + "label": "Balanced \u2014 stock boost, power unlocked", + "power_limit_w": 370, + "core_offset_mhz": 0, + "mem_offset_mhz": 0, + "lock_core_min": 0, + "lock_core_max": 0, + "lock_mem_mhz": 0, + "fan_mode": "auto", + "fan_speed_pct": 0 + } +} \ No newline at end of file diff --git a/server.py b/server.py index 558a84a..5ebfb7f 100644 --- a/server.py +++ b/server.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, Field import ram_optimizer import vram_arbitrator +import overclock_manager logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s") logger = logging.getLogger("model_manager_server") @@ -31,6 +32,14 @@ app.add_middleware( allow_headers=["*"], ) +@app.on_event("startup") +async def on_startup(): + await vram_arbitrator.arbitrator.start() + +@app.on_event("shutdown") +async def on_shutdown(): + await vram_arbitrator.arbitrator.stop() + # Pydantic Request Models class SwitchRequest(BaseModel): model: str = Field(..., description="Name of the Ollama model to hot-swap to in VRAM", example="qwen3.8fast:latest") @@ -44,6 +53,17 @@ class BenchmarkRequest(BaseModel): iterations: Optional[int] = Field(2, description="Number of back-and-forth switch iterations to measure", example=2) models: Optional[List[str]] = Field(None, description="Optional pair of models to benchmark between", example=["qwen3.8fast:latest", "smtek/Qwen3.8-27B:Q2_K_XL"]) +class OverclockApplyRequest(BaseModel): + profile: str = Field(..., description="Profile name: ollama | comfy | balanced", example="ollama") + +class OverclockProfileUpdate(BaseModel): + config: Dict[str, Any] = Field(..., description="Profile settings dict", example={"power_limit_w": 370, "core_offset_mhz": 100}) + +class FanRequest(BaseModel): + mode: str = Field("auto", description="'auto' or 'manual'", example="manual") + percent: Optional[int] = Field(None, description="Fan speed 30-100 when mode=manual", example=70) + speed_pct: Optional[int] = Field(None, description="Alias for percent (30-100)", example=70) + # ========================================== # REST API ENDPOINTS @@ -58,6 +78,7 @@ async def get_all_stats() -> Dict[str, Any]: comfy_state = await vram_arbitrator.get_comfyui_live_state() history = vram_arbitrator.get_switch_history() comfy_models = ram_optimizer.find_comfy_model_files() + arbitrator_status = vram_arbitrator.arbitrator.get_status() return { "timestamp": asyncio.get_event_loop().time(), @@ -65,10 +86,12 @@ async def get_all_stats() -> Dict[str, Any]: "ram": mem_stats, "ollama": ollama_state, "comfyui": comfy_state, + "arbitrator": arbitrator_status, "history": history, "comfy_models_count": len(comfy_models), } + @app.get("/api/gpu", summary="GPU Sensors and VRAM Breakdown", tags=["Telemetry"]) async def get_gpu_metrics() -> Dict[str, Any]: """Retrieve detailed NVML sensors (utilization %, temp, power, fan, clocks, and per-process VRAM allocation).""" @@ -160,6 +183,51 @@ async def api_run_benchmark(req: BenchmarkRequest): res_str = await run_model_switch_benchmark(iterations=req.iterations or 2) return json.loads(res_str) +# ========================================== +# OVERCLOCK MANAGEMENT +# ========================================== + +@app.get("/api/overclock", summary="Overclock Status & Profiles", tags=["Overclock"]) +async def api_overclock_status(): + """Get live GPU overclock state, active profile, and all per-app profiles.""" + return overclock_manager.get_status() + +@app.post("/api/overclock/apply", summary="Apply Overclock Profile", tags=["Overclock"]) +async def api_overclock_apply(req: OverclockApplyRequest): + """Apply a named overclock profile (ollama | comfy | balanced) to the GPU immediately.""" + res = overclock_manager.apply_profile(req.profile) + if not res.get("success"): + raise HTTPException(status_code=400, detail=res.get("error")) + return res + +@app.get("/api/overclock/profiles", summary="List Overclock Profiles", tags=["Overclock"]) +async def api_overclock_profiles(): + """List all overclock profiles with their current settings.""" + return overclock_manager.get_profiles() + +@app.post("/api/overclock/profiles/{name}", summary="Update Overclock Profile", tags=["Overclock"]) +async def api_overclock_update_profile(name: str, req: OverclockProfileUpdate): + """Update a profile's settings (persisted to disk).""" + res = overclock_manager.set_profile(name, req.config) + if not res.get("success"): + raise HTTPException(status_code=400, detail=res.get("error")) + return {"success": True, "profile": name, "profiles": res.get("profiles")} + +@app.get("/api/overclock/fan", summary="Get GPU Fan Status", tags=["Overclock"]) +@app.get("/api/gpu/fan", summary="Get GPU Fan Status", tags=["Overclock"]) +async def api_get_fan_status(): + """Get current GPU fan control mode and speed.""" + return overclock_manager.get_fan_status() + +@app.post("/api/overclock/fan", summary="Set GPU Fan Speed", tags=["Overclock"]) +@app.post("/api/gpu/fan", summary="Set GPU Fan Speed", tags=["Overclock"]) +async def api_set_fan(req: FanRequest): + """Set the GPU fan to manual speed (30-100%) or back to automatic control.""" + pct = req.percent if req.percent is not None else req.speed_pct + if req.mode == "manual" and pct is not None: + return overclock_manager.set_fan_speed(pct) + return overclock_manager.set_fan_auto() + # Mount static web UI files app.mount("/static", StaticFiles(directory="/home/drjones/unified-model-manager/static"), name="static") diff --git a/static/app.js b/static/app.js index 76d2082..3cdbf22 100644 --- a/static/app.js +++ b/static/app.js @@ -36,6 +36,7 @@ function updateDashboard(data) { // 1. GPU VRAM Stats const gpu = data.gpu || {}; if (gpu.available) { + pushOverclockSample(gpu); document.getElementById('gpu-chip-name').textContent = gpu.device_name || 'NVIDIA GPU'; document.getElementById('vram-total-used').textContent = gpu.vram_used_gb || '0.0'; document.getElementById('vram-used-pct').textContent = `${gpu.vram_used_pct || 0}% USED`; @@ -68,6 +69,45 @@ function updateDashboard(data) { document.getElementById('gpu-power-val').textContent = `${gpu.power_w || 0} W`; document.getElementById('gpu-fan-val').textContent = `${gpu.fan_pct || 0}%`; + // Per-fan and animations + const fan0 = (gpu.fans && gpu.fans.length > 0) ? gpu.fans[0] : (gpu.fan_pct || 0); + const fan1 = (gpu.fans && gpu.fans.length > 1) ? gpu.fans[1] : (gpu.fan_pct || 0); + const fanSub = document.getElementById('gpu-fan-sub'); + if (fanSub) fanSub.textContent = `Fan 0: ${fan0}% | Fan 1: ${fan1}%`; + + const fan0Val = document.getElementById('oc-fan0-val'); + if (fan0Val) fan0Val.textContent = `${fan0}%`; + const fan0Bar = document.getElementById('oc-fan0-bar'); + if (fan0Bar) fan0Bar.style.width = `${fan0}%`; + + const fan1Val = document.getElementById('oc-fan1-val'); + if (fan1Val) fan1Val.textContent = `${fan1}%`; + const fan1Bar = document.getElementById('oc-fan1-bar'); + if (fan1Bar) fan1Bar.style.width = `${fan1}%`; + + const fanCurrent = document.getElementById('oc-fan-current'); + if (fanCurrent) fanCurrent.textContent = `${gpu.fan_pct || 0}%`; + + const spinSpeed = Math.max(0.2, (100 - (gpu.fan_pct || 0)) / 100 * 1.6 + 0.3); + const fanIcon = document.getElementById('gpu-fan-icon'); + if (fanIcon) { + if ((gpu.fan_pct || 0) > 0) { + fanIcon.classList.add('fan-spinning'); + fanIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`; + } else { + fanIcon.classList.remove('fan-spinning'); + } + } + const ocFanCardIcon = document.getElementById('oc-fan-card-icon'); + if (ocFanCardIcon) { + if ((gpu.fan_pct || 0) > 0) { + ocFanCardIcon.classList.add('fan-spinning'); + ocFanCardIcon.style.animationDuration = `${spinSpeed.toFixed(2)}s`; + } else { + ocFanCardIcon.classList.remove('fan-spinning'); + } + } + // Processes table const tbody = document.getElementById('gpu-proc-table'); if (bd.processes && bd.processes.length > 0) { @@ -285,4 +325,287 @@ async function warmAllModels() { // Startup document.addEventListener('DOMContentLoaded', () => { initSSE(); + initOverclockChart(); + fetchOverclockStatus(); + setInterval(fetchOverclockStatus, 3000); }); + +// ============ OVERCLOCK CONTROL ============ +let ocProfiles = {}; +let ocChart = null; +const OC_MAX_SAMPLES = 120; + +function initOverclockChart() { + const canvas = document.getElementById('oc-chart'); + if (!canvas || typeof Chart === 'undefined') return; + const ctx = canvas.getContext('2d'); + + ocChart = new Chart(ctx, { + type: 'line', + data: { + labels: [], + datasets: [ + { label: 'Core MHz', data: [], borderColor: '#22d3ee', backgroundColor: 'rgba(34,211,238,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: true }, + { label: 'Mem MHz', data: [], borderColor: '#c084fc', backgroundColor: 'rgba(192,132,252,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y', fill: false }, + { label: 'Temp °C', data: [], borderColor: '#fb7185', backgroundColor: 'rgba(251,113,133,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y1', fill: false }, + { label: 'Power W', data: [], borderColor: '#fbbf24', backgroundColor: 'rgba(251,191,36,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y2', fill: false }, + { label: 'Fan %', data: [], borderColor: '#34d399', backgroundColor: 'rgba(52,211,153,0.08)', borderWidth: 1.5, pointRadius: 0, tension: 0.35, yAxisID: 'y3', fill: false }, + ] + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + interaction: { mode: 'index', intersect: false }, + scales: { + x: { ticks: { color: '#64748b', maxTicksLimit: 8, font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } }, + y: { position: 'left', title: { display: true, text: 'MHz', color: '#22d3ee', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { color: 'rgba(51,65,85,0.35)' } }, + y1: { position: 'right', title: { display: true, text: '°C', color: '#fb7185', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 }, + y2: { position: 'right', offset: true, title: { display: true, text: 'W', color: '#fbbf24', font: { size: 9 } }, ticks: { color: '#64748b', font: { size: 9 } }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 400 }, + y3: { position: 'right', offset: true, title: { display: false }, ticks: { display: false }, grid: { drawOnChartArea: false }, suggestedMin: 0, suggestedMax: 100 }, + }, + plugins: { legend: { display: false } }, + } + }); +} + +function pushOverclockSample(gpu) { + if (!ocChart || !gpu || !gpu.available) return; + const label = new Date().toLocaleTimeString([], { hour12: false }); + ocChart.data.labels.push(label); + ocChart.data.datasets[0].data.push(gpu.clock_graphics_mhz || 0); + ocChart.data.datasets[1].data.push(gpu.clock_mem_mhz || 0); + ocChart.data.datasets[2].data.push(gpu.temperature_c || 0); + ocChart.data.datasets[3].data.push(gpu.power_w || 0); + ocChart.data.datasets[4].data.push(gpu.fan_pct || 0); + if (ocChart.data.labels.length > OC_MAX_SAMPLES) { + ocChart.data.labels.shift(); + ocChart.data.datasets.forEach(d => d.data.shift()); + } + ocChart.update('none'); +} + +async function fetchOverclockStatus() { + try { + const resp = await fetch('/api/overclock'); + if (!resp.ok) return; + const data = await resp.json(); + ocProfiles = data.profiles || {}; + renderOverclockStatus(data); + } catch (err) { + console.warn('Overclock fetch error:', err); + } +} + +function renderOverclockStatus(data) { + const active = data.active_profile || 'balanced'; + + const badge = document.getElementById('oc-active-badge'); + badge.textContent = `Active: ${active}`; + if (active === 'ollama') { + badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-purple-950 border border-purple-700 text-purple-300'; + } else if (active === 'comfy') { + badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-cyan-950 border border-cyan-700 text-cyan-300'; + } else { + badge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-300'; + } + + const gpu = data.gpu || {}; + document.getElementById('oc-power-limit').textContent = `${gpu.power_limit_w ?? '--'} W`; + document.getElementById('oc-core').textContent = `${gpu.clock_sm_mhz ?? '--'} MHz`; + document.getElementById('oc-mem').textContent = `${gpu.clock_mem_mhz ?? '--'} MHz`; + document.getElementById('oc-temp-draw').textContent = `${gpu.temp_c ?? '--'}°C / ${gpu.power_draw_w ?? '--'}W`; + + // Specs strip + document.getElementById('oc-spec-gpu').textContent = gpu.name || '--'; + document.getElementById('oc-spec-driver').textContent = gpu.driver_version || '--'; + const vramGb = (gpu.vram_total_mb || 0) / 1024; + document.getElementById('oc-spec-vram').textContent = vramGb > 0 ? vramGb.toFixed(0) + ' GB' : '--'; + document.getElementById('oc-spec-maxcore').textContent = (gpu.clock_sm_max_mhz ?? '--') + ' MHz'; + document.getElementById('oc-spec-maxmem').textContent = (gpu.clock_mem_max_mhz ?? '--') + ' MHz'; + document.getElementById('oc-spec-power').textContent = `${gpu.power_limit_w ?? '--'} / ${gpu.power_max_w ?? '--'} W`; + + // Fan status + const fan = data.fan || {}; + const fanBadge = document.getElementById('oc-fan-badge'); + if (fanBadge) { + if (fan.manual) { + fanBadge.textContent = `MANUAL (${fan.target_speed_pct ?? '--'}%)`; + fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-500 text-emerald-300 font-bold'; + } else { + fanBadge.textContent = 'AUTO (VBIOS)'; + fanBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-slate-800 border border-slate-700 text-slate-400 font-bold'; + } + } + + const xBadge = document.getElementById('oc-x-badge'); + if (data.headless_x_running) { + xBadge.textContent = 'X: ON'; + xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-emerald-950 border border-emerald-700 text-emerald-300'; + } else { + xBadge.textContent = 'X: OFF'; + xBadge.className = 'px-2.5 py-1 text-xs font-mono rounded-lg bg-rose-950 border border-rose-700 text-rose-300'; + } + + const offsets = (data.last_result && data.last_result.offsets) || {}; + const note = document.getElementById('oc-offset-note'); + if (offsets.supported) { + note.textContent = '✅ Clock offsets active (proprietary driver)'; + note.className = 'text-[11px] font-mono text-emerald-400'; + } else { + note.textContent = '⚠️ Clock offsets off — nvidia-open lacks offset support (power + clock locks active)'; + note.className = 'text-[11px] font-mono text-amber-400'; + } + + // highlight active profile button + ['ollama', 'comfy', 'balanced'].forEach(p => { + const btn = document.getElementById('oc-btn-' + p); + if (p === active) { + btn.classList.add('ring-2', 'ring-fuchsia-400'); + } else { + btn.classList.remove('ring-2', 'ring-fuchsia-400'); + } + }); +} + +function loadOverclockForProfile(name) { + const p = ocProfiles[name]; + if (!p) return; + document.getElementById('oc-slider-power').value = p.power_limit_w || 370; + document.getElementById('oc-val-power').textContent = (p.power_limit_w || 370) + ' W'; + document.getElementById('oc-slider-core').value = p.core_offset_mhz || 0; + document.getElementById('oc-val-core').textContent = '+' + (p.core_offset_mhz || 0) + ' MHz'; + document.getElementById('oc-slider-mem').value = p.mem_offset_mhz || 0; + document.getElementById('oc-val-mem').textContent = '+' + (p.mem_offset_mhz || 0) + ' MHz'; + document.getElementById('oc-lockcore-toggle').checked = (p.lock_core_max || 0) > 0; + document.getElementById('oc-val-lockcore').textContent = (p.lock_core_max || 0) > 0 ? 'On' : 'Off'; + document.getElementById('oc-lockmem-toggle').checked = (p.lock_mem_mhz || 0) > 0; + document.getElementById('oc-val-lockmem').textContent = (p.lock_mem_mhz || 0) > 0 ? 'On' : 'Off'; + + const fanMode = p.fan_mode || 'auto'; + const profFanMode = document.getElementById('oc-prof-fanmode'); + if (profFanMode) profFanMode.value = fanMode; + const profFanSpeed = document.getElementById('oc-slider-prof-fanspeed'); + if (profFanSpeed) profFanSpeed.value = p.fan_speed_pct || 70; + const profFanVal = document.getElementById('oc-val-prof-fanspeed'); + if (profFanVal) profFanVal.textContent = (p.fan_speed_pct || 70) + '%'; + toggleProfileFanMode(); +} + +function toggleProfileFanMode() { + const mode = document.getElementById('oc-prof-fanmode').value; + const container = document.getElementById('oc-prof-fanspeed-container'); + const valBadge = document.getElementById('oc-val-prof-fanmode'); + if (valBadge) valBadge.textContent = (mode === 'manual') ? 'Manual Target' : 'Auto (VBIOS)'; + if (container) { + if (mode === 'manual') { + container.classList.remove('opacity-50', 'pointer-events-none'); + } else { + container.classList.add('opacity-50', 'pointer-events-none'); + } + } +} + +function toggleCoreLock() { + document.getElementById('oc-val-lockcore').textContent = document.getElementById('oc-lockcore-toggle').checked ? 'On' : 'Off'; +} + +function toggleMemLock() { + document.getElementById('oc-val-lockmem').textContent = document.getElementById('oc-lockmem-toggle').checked ? 'On' : 'Off'; +} + +async function setFanManual(speed) { + let pct = speed; + if (pct === undefined || pct === null) { + pct = parseInt(document.getElementById('oc-fan-slider').value); + } else { + pct = parseInt(pct); + const slider = document.getElementById('oc-fan-slider'); + if (slider) slider.value = pct; + const label = document.getElementById('oc-fan-slider-label'); + if (label) label.textContent = pct + '%'; + } + try { + const resp = await fetch('/api/overclock/fan', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode: 'manual', percent: pct }) + }); + const result = await resp.json(); + if (!resp.ok) alert(`Fan set failed: ${result.detail || 'error'}`); + await fetchOverclockStatus(); + } catch (err) { + alert(`Error setting fan speed: ${err}`); + } +} + +async function setFanAuto() { + try { + const resp = await fetch('/api/overclock/fan', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode: 'auto' }) + }); + const result = await resp.json(); + if (!resp.ok) alert(`Fan auto failed: ${result.detail || 'error'}`); + await fetchOverclockStatus(); + } catch (err) { + alert(`Error setting fan to auto: ${err}`); + } +} + +async function applyOverclock(profile) { + try { + const resp = await fetch('/api/overclock/apply', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ profile }) + }); + const result = await resp.json(); + if (!resp.ok) { + alert(`Apply failed: ${result.detail || 'error'}`); + } + await fetchOverclockStatus(); + } catch (err) { + alert(`Error: ${err}`); + } +} + +async function saveOverclockProfile() { + const name = document.getElementById('oc-edit-profile').value; + const power = parseInt(document.getElementById('oc-slider-power').value); + const core = parseInt(document.getElementById('oc-slider-core').value); + const mem = parseInt(document.getElementById('oc-slider-mem').value); + const lockCore = document.getElementById('oc-lockcore-toggle').checked; + const lockMem = document.getElementById('oc-lockmem-toggle').checked; + const fanMode = document.getElementById('oc-prof-fanmode') ? document.getElementById('oc-prof-fanmode').value : 'auto'; + const fanSpeed = document.getElementById('oc-slider-prof-fanspeed') ? parseInt(document.getElementById('oc-slider-prof-fanspeed').value) : 70; + + const config = { + power_limit_w: power, + core_offset_mhz: core, + mem_offset_mhz: mem, + lock_core_min: lockCore ? 2900 : 0, + lock_core_max: lockCore ? 3105 : 0, + lock_mem_mhz: lockMem ? 11501 : 0, + fan_mode: fanMode, + fan_speed_pct: (fanMode === 'manual') ? fanSpeed : 0, + }; + + try { + const resp = await fetch(`/api/overclock/profiles/${name}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ config }) + }); + const result = await resp.json(); + if (resp.ok) { + await fetchOverclockStatus(); + await applyOverclock(name); + } else { + alert(`Save failed: ${result.detail || 'error'}`); + } + } catch (err) { + alert(`Error: ${err}`); + } +} diff --git a/static/index.html b/static/index.html index 0a4c2d6..c9b5e01 100644 --- a/static/index.html +++ b/static/index.html @@ -5,6 +5,7 @@ HYPERSWAP // Dual-Engine Model Orchestrator & Live Telemetry + @@ -351,9 +352,13 @@ Power Draw 0 W -
+
Fan Speed - 0% +
+ + 0% +
+ Fan 0: --% | Fan 1: --%
@@ -419,6 +424,245 @@ + +
+
+
+
+ +
+
+

GPU Overclock Control

+

Per-app profiles auto-switch with the VRAM arbitrator

+
+
+
+ Active: -- + X: -- +
+
+ + +
+ + + +
+ + +
+
+ Power Limit + -- W +
+
+ Core Clock + -- +
+
+ Mem Clock + -- +
+
+ Temp / Draw + -- +
+
+ + +
+
+
+ +
+ GPU Fan Cooling Control + Dual-fan PWM active speed regulation +
+
+ + AUTO (VBIOS) + +
+ + +
+ + + + + +
+ + +
+
+
+ Fan 0 (Intake/Core): + --% +
+
+
+
+
+
+
+ Fan 1 (Exhaust/VRM): + --% +
+
+
+
+
+
+ + +
+
+ Custom Manual Target: + 65% +
+
+ + +
+
+
+ + +
+
+ GPU + -- +
+
+ Driver + -- +
+
+ VRAM + -- +
+
+ Max Core + -- +
+
+ Max Mem + -- +
+
+ Power + -- +
+
+ + +
+
+ + Live Tuning Graph + +
+ Core MHz + Mem MHz + Temp °C + Power W + Fan % +
+
+
+ +
+
+ + +
+
+ Fine-tune profile + +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ +
+ + Lock core to max boost (2900–3105 MHz) +
+
+
+ +
+ + Lock mem to max (11501 MHz) +
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + +
+
+
+ diff --git a/static/styles.css b/static/styles.css index f5c8b31..19a82b1 100644 --- a/static/styles.css +++ b/static/styles.css @@ -26,3 +26,13 @@ .glow-card { animation: pulseGlow 4s infinite ease-in-out; } + +@keyframes spinFan { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +.fan-spinning { + display: inline-block; + animation: spinFan 1s linear infinite; +} diff --git a/vram_arbitrator.py b/vram_arbitrator.py index be99120..f1fcea2 100644 --- a/vram_arbitrator.py +++ b/vram_arbitrator.py @@ -6,6 +6,12 @@ import logging from typing import Dict, List, Any, Optional from collections import deque +import asyncio +import json +import websockets + +import overclock_manager + try: import pynvml pynvml.nvmlInit() @@ -21,6 +27,7 @@ COMFY_API_BASE = "http://127.0.0.1:8188" # Circular buffer for transition events SWITCH_HISTORY = deque(maxlen=50) + def get_gpu_hardware_stats() -> Dict[str, Any]: """Retrieve comprehensive GPU hardware and process metrics via NVML.""" if not NVML_AVAILABLE: @@ -41,10 +48,27 @@ def get_gpu_hardware_stats() -> Dict[str, Any]: except Exception: power_w = 0.0 + fan_pct = 0 + fans = [] try: - fan_pct = pynvml.nvmlDeviceGetFanSpeed(handle) + num_fans = pynvml.nvmlDeviceGetNumFans(handle) + for i in range(num_fans): + try: + fans.append(pynvml.nvmlDeviceGetFanSpeed_v2(handle, i)) + except Exception: + pass + if fans: + fan_pct = max(fans) + else: + fan_pct = pynvml.nvmlDeviceGetFanSpeed(handle) + fans = [fan_pct] except Exception: - fan_pct = 0 + try: + fan_pct = pynvml.nvmlDeviceGetFanSpeed(handle) + fans = [fan_pct] + except Exception: + fan_pct = 0 + fans = [] try: clock_graphics = pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_GRAPHICS) @@ -119,6 +143,8 @@ def get_gpu_hardware_stats() -> Dict[str, Any]: "temperature_c": temp_c, "power_w": power_w, "fan_pct": fan_pct, + "fans": fans, + "num_fans": len(fans), "clock_graphics_mhz": clock_graphics, "clock_mem_mhz": clock_mem, "breakdown": { @@ -320,3 +346,159 @@ async def switch_ollama_model(target_model: str, keep_alive: str = "30m") -> Dic def get_switch_history() -> List[Dict[str, Any]]: return list(SWITCH_HISTORY) + + +class AutoArbitrator: + """Real-time bidirectional background arbitrator for seamless Ollama <-> ComfyUI hot-swapping.""" + + def __init__(self): + self.running = False + self.ws_task: Optional[asyncio.Task] = None + self.poll_task: Optional[asyncio.Task] = None + self.last_yield_time = 0.0 + self.last_comfy_free_time = 0.0 + self.connected_ws = False + self.last_action = "Idle" + self.comfy_was_active = False + self.oc_profile = None + + async def start(self): + if self.running: + return + self.running = True + self.ws_task = asyncio.create_task(self._ws_listener()) + self.poll_task = asyncio.create_task(self._poll_watchdog()) + logger.info("AutoArbitrator background engine started (Bidirectional).") + # Apply the default (balanced) overclock profile on startup. + try: + await asyncio.get_event_loop().run_in_executor(None, overclock_manager.apply_profile, "balanced") + except Exception as e: + logger.warning(f"Startup overclock apply failed: {e}") + + async def stop(self): + self.running = False + if self.ws_task: + self.ws_task.cancel() + if self.poll_task: + self.poll_task.cancel() + logger.info("AutoArbitrator background engine stopped.") + + async def trigger_comfy_priority(self, reason: str = "ComfyUI prompt detected"): + """Instantly yield Ollama VRAM to 0MB when ComfyUI needs to run diffusion models.""" + self.comfy_was_active = True + self._apply_oc_profile("comfy") + now = time.time() + if now - self.last_yield_time < 1.0: + return + + ollama_state = await get_ollama_live_state() + if ollama_state.get("active_model_name"): + model = ollama_state["active_model_name"] + logger.info(f"⚡ ComfyUI active ({reason}) -> Auto-yielding Ollama model '{model}' from VRAM...") + self.last_yield_time = time.time() + res = await instant_free_ollama_vram(model) + dur = res.get("duration_ms", 0) + self.last_action = f"Auto-yielded '{model}' for ComfyUI ({dur}ms)" + logger.info(f"Ollama auto-yield completed: {res}") + + async def trigger_comfy_completed(self): + """Purge ComfyUI VRAM cache when generation finishes, keeping VRAM 100% free for Ollama.""" + now = time.time() + if now - self.last_comfy_free_time < 3.0: + return + self.last_comfy_free_time = now + self.comfy_was_active = False + self._apply_oc_profile("ollama") + logger.info("⚡ ComfyUI finished generation -> Auto-purging ComfyUI VRAM cache for Ollama...") + res = await instant_free_comfyui_vram() + dur = res.get("duration_ms", 0) + self.last_action = f"Auto-purged ComfyUI VRAM ({dur}ms) - Ready for Ollama" + logger.info(f"ComfyUI auto-purge completed: {res}") + + async def _ws_listener(self): + client_id = "hyperswap-arbitrator" + ws_url = f"ws://127.0.0.1:8188/ws?clientId={client_id}" + + while self.running: + try: + async with websockets.connect(ws_url, ping_interval=10, ping_timeout=10) as ws: + self.connected_ws = True + logger.info("AutoArbitrator connected to ComfyUI WebSocket.") + while self.running: + msg = await ws.recv() + if isinstance(msg, str): + try: + data = json.loads(msg) + msg_type = data.get("type") + msg_data = data.get("data", {}) + + if msg_type == "status": + queue_rem = msg_data.get("status", {}).get("exec_info", {}).get("queue_remaining", 0) + if queue_rem > 0: + await self.trigger_comfy_priority(f"Queue remaining: {queue_rem}") + elif queue_rem == 0 and self.comfy_was_active: + # Prompt queue finished + await asyncio.sleep(1.5) + await self.trigger_comfy_completed() + elif msg_type in ("execution_start", "execution_cached"): + await self.trigger_comfy_priority(f"Event: {msg_type}") + elif msg_type == "executing": + node = msg_data.get("node") + if node is not None: + await self.trigger_comfy_priority(f"Executing node: {node}") + elif node is None and self.comfy_was_active: + # Finished executing graph + await asyncio.sleep(1.5) + await self.trigger_comfy_completed() + elif msg_type == "execution_success": + await asyncio.sleep(1.5) + await self.trigger_comfy_completed() + except Exception as e: + logger.debug(f"WS parse error: {e}") + except (websockets.exceptions.ConnectionClosed, OSError, asyncio.CancelledError): + self.connected_ws = False + except Exception as e: + self.connected_ws = False + logger.debug(f"WS connection error: {e}") + + await asyncio.sleep(2.0) + + async def _poll_watchdog(self): + """Watchdog polling /queue every 300ms for robust bidirectional arbitration.""" + while self.running: + try: + comfy_state = await get_comfyui_live_state() + is_executing = comfy_state.get("queue_running", 0) > 0 or comfy_state.get("queue_remaining", 0) > 0 or comfy_state.get("executing", False) + if is_executing: + await self.trigger_comfy_priority("Polling detected active queue/execution") + elif self.comfy_was_active and not is_executing: + await asyncio.sleep(1.5) + await self.trigger_comfy_completed() + except Exception: + pass + await asyncio.sleep(0.3) + + def _apply_oc_profile(self, profile: str): + """Apply an overclock profile in a background thread; only fire on transition.""" + if self.oc_profile == profile: + return + self.oc_profile = profile + try: + loop = asyncio.get_event_loop() + loop.run_in_executor(None, overclock_manager.apply_profile, profile) + logger.info(f"🎛️ Overclock profile switched -> '{profile}'") + except Exception as e: + logger.warning(f"Overclock profile switch failed ({profile}): {e}") + + def get_status(self) -> Dict[str, Any]: + return { + "running": self.running, + "connected_ws": self.connected_ws, + "last_action": self.last_action, + "mode": "Bidirectional Hot-Swap (ComfyUI <-> Ollama)", + } + + +arbitrator = AutoArbitrator() + +