node agent v2: spawn/destroy GPU Ubuntu sessions (noVNC + 4080 passthrough via docker --gpus all)

This commit is contained in:
drjones
2026-08-24 21:13:56 -07:00
parent 28b2c7339f
commit d0f81b7432

View File

@@ -1,17 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
VORTEX_GPU — Linux Host Node Agent (port of vortex-node-agent.ps1) VORTEX_GPU — Linux Host Node Agent (v2 — Ubuntu-session provisioning)
Target: Ubuntu (nightmare .128, RTX 4080 SUPER 16GB) Target: Ubuntu (nightmare .128, RTX 4080 SUPER 16GB)
Responsibilities (mirrors the Windows agent): Spawns in-browser Ubuntu desktop sessions, each with the 4080 attached (--gpus all):
- Register this GPU box + stream nvidia-smi telemetry every N sec. - provision_ubuntu : docker run a full Ubuntu LXDE desktop (noVNC) with GPU, on a
- Poll the gateway for jobs: dedicated port. Tenant gets a clean private machine; the physical
* shell : run an arbitrary cmd, return stdout GPU is shared/hidden.
* hashcat / comfyui : run the command against the local GPU - destroy_ubuntu : docker rm -f the session container.
* provision_comfyui : launch an ISOLATED ComfyUI instance on a dedicated - shell / hashcat / comfyui : run an arbitrary command against the local GPU.
port + dedicated user/output/input dirs (tenant sees
a clean private machine; the physical GPU is shared/hidden)
* destroy_instance : kill the ComfyUI process for an instance
Auth: X-Node-Secret header (matches server.ts nodeAuthorized). Auth: X-Node-Secret header (matches server.ts nodeAuthorized).
Runs as a systemd service: vortex-node-agent.service Runs as a systemd service: vortex-node-agent.service
@@ -28,13 +25,8 @@ SECRET = os.environ.get("VORTEX_NODE_SECRET", "99496a5bf30b5a7411d3a60bf096ca6
INTERVAL = int(os.environ.get("VORTEX_INTERVAL", "5")) INTERVAL = int(os.environ.get("VORTEX_INTERVAL", "5"))
HOSTNAME = socket.gethostname() HOSTNAME = socket.gethostname()
SESSION_IMAGE = os.environ.get("VORTEX_SESSION_IMAGE", "dorowu/ubuntu-desktop-lxde-vnc:latest")
HOME = os.path.expanduser("~") HOME = os.path.expanduser("~")
COMFY_DIR = os.path.join(HOME, "ComfyUI")
COMFY_PY = os.path.join(COMFY_DIR, "venv", "bin", "python")
INST_DIR = os.path.join(HOME, "vortex-agent", "instances")
os.makedirs(INST_DIR, exist_ok=True)
_instance_pids = {} # instanceId -> pid
def http(method, path, body=None): def http(method, path, body=None):
@@ -106,6 +98,10 @@ def uptime_sec():
return int(float(open("/proc/uptime").read().split()[0])) return int(float(open("/proc/uptime").read().split()[0]))
def _docker(args, timeout=180):
return subprocess.run(["docker"] + args, capture_output=True, text=True, timeout=timeout)
def run_shell(command): def run_shell(command):
try: try:
r = subprocess.run(["bash", "-c", command], capture_output=True, text=True, timeout=600) r = subprocess.run(["bash", "-c", command], capture_output=True, text=True, timeout=600)
@@ -114,46 +110,25 @@ def run_shell(command):
return False, f"error: {e}" return False, f"error: {e}"
def provision_comfyui(instance_id, port, user_dir): def provision_ubuntu(instance_id, port, password, resolution):
os.makedirs(user_dir, exist_ok=True) """Spawn a full Ubuntu desktop session with the 4080 attached (noVNC on mapped port)."""
for sub in ("user", "output", "input", "models", "custom_nodes"): name = f"vortex-{instance_id}"
os.makedirs(os.path.join(user_dir, sub), exist_ok=True) _docker(["rm", "-f", name], timeout=30) # clean any stale instance
if not os.path.exists(os.path.join(COMFY_DIR, "main.py")): r = _docker(["run", "-d", "--gpus", "all", "--name", name,
return False, "ComfyUI base not found on this node" "-p", f"{port}:80",
if not os.path.exists(COMFY_PY): "-e", f"VNC_PASSWORD={password}",
return False, "ComfyUI venv not found" "-e", f"RESOLUTION={resolution or '1440x900'}",
args = [COMFY_PY, os.path.join(COMFY_DIR, "main.py"), SESSION_IMAGE])
"--listen", "0.0.0.0", "--port", str(port), if r.returncode == 0:
"--user-directory", os.path.join(user_dir, "user"), cid = r.stdout.strip()[:12]
"--output-directory", os.path.join(user_dir, "output"), return True, f"launched container={name} id={cid} port={port}"
"--input-directory", os.path.join(user_dir, "input")] return False, f"failed: {r.stderr.strip()[:400]}"
try:
proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True)
_instance_pids[instance_id] = proc.pid
with open(os.path.join(user_dir, "instance.pid"), "w") as f:
f.write(str(proc.pid))
return True, f"launched pid={proc.pid} port={port}"
except Exception as e:
return False, f"failed to launch: {e}"
def destroy_instance(instance_id): def destroy_ubuntu(instance_id):
pid = _instance_pids.pop(instance_id, None) name = f"vortex-{instance_id}"
if pid is None: r = _docker(["rm", "-f", name], timeout=30)
pidfile = os.path.join(INST_DIR, instance_id, "instance.pid") return True, (r.stdout.strip() or f"removed {name}")
if os.path.exists(pidfile):
try:
pid = int(open(pidfile).read().strip())
except Exception:
pid = None
if pid:
try:
os.kill(pid, 15)
return True, f"killed pid={pid}"
except Exception as e:
return True, f"kill pid={pid}: {e}"
return True, "no pid found"
def handle_job(job): def handle_job(job):
@@ -162,14 +137,15 @@ def handle_job(job):
payload = job.get("payload", {}) or {} payload = job.get("payload", {}) or {}
if kind in ("shell", "hashcat", "comfyui"): if kind in ("shell", "hashcat", "comfyui"):
ok, result = run_shell(cmd) ok, result = run_shell(cmd)
elif kind == "provision_comfyui": elif kind == "provision_ubuntu":
ok, result = provision_comfyui(payload.get("instanceId", "inst"), ok, result = provision_ubuntu(payload.get("instanceId", "inst"),
payload.get("port", 8189), int(payload.get("port", 6090)),
payload.get("userDir") or os.path.join(INST_DIR, payload.get("instanceId", "inst"))) payload.get("password", "vortex"),
elif kind == "destroy_instance": payload.get("resolution", "1440x900"))
ok, result = destroy_instance(payload.get("instanceId", "")) elif kind == "destroy_ubuntu":
ok, result = destroy_ubuntu(payload.get("instanceId", ""))
else: else:
ok, result = False, "unknown job kind" ok, result = False, f"unknown job kind: {kind}"
try: try:
http("POST", f"/api/node/jobs/{job['id']}/result", {"ok": ok, "result": result}) http("POST", f"/api/node/jobs/{job['id']}/result", {"ok": ok, "result": result})
except Exception as e: except Exception as e: