node agent v2: spawn/destroy GPU Ubuntu sessions (noVNC + 4080 passthrough via docker --gpus all)
This commit is contained in:
@@ -1,17 +1,14 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
VORTEX_GPU — Linux Host Node Agent (port of vortex-node-agent.ps1)
|
VORTEX_GPU — Linux Host Node Agent (v2 — Ubuntu-session provisioning)
|
||||||
Target: Ubuntu (nightmare .128, RTX 4080 SUPER 16GB)
|
Target: Ubuntu (nightmare .128, RTX 4080 SUPER 16GB)
|
||||||
|
|
||||||
Responsibilities (mirrors the Windows agent):
|
Spawns in-browser Ubuntu desktop sessions, each with the 4080 attached (--gpus all):
|
||||||
- Register this GPU box + stream nvidia-smi telemetry every N sec.
|
- provision_ubuntu : docker run a full Ubuntu LXDE desktop (noVNC) with GPU, on a
|
||||||
- Poll the gateway for jobs:
|
dedicated port. Tenant gets a clean private machine; the physical
|
||||||
* shell : run an arbitrary cmd, return stdout
|
GPU is shared/hidden.
|
||||||
* hashcat / comfyui : run the command against the local GPU
|
- destroy_ubuntu : docker rm -f the session container.
|
||||||
* provision_comfyui : launch an ISOLATED ComfyUI instance on a dedicated
|
- shell / hashcat / comfyui : run an arbitrary command against the local GPU.
|
||||||
port + dedicated user/output/input dirs (tenant sees
|
|
||||||
a clean private machine; the physical GPU is shared/hidden)
|
|
||||||
* destroy_instance : kill the ComfyUI process for an instance
|
|
||||||
|
|
||||||
Auth: X-Node-Secret header (matches server.ts nodeAuthorized).
|
Auth: X-Node-Secret header (matches server.ts nodeAuthorized).
|
||||||
Runs as a systemd service: vortex-node-agent.service
|
Runs as a systemd service: vortex-node-agent.service
|
||||||
@@ -28,13 +25,8 @@ SECRET = os.environ.get("VORTEX_NODE_SECRET", "99496a5bf30b5a7411d3a60bf096ca6
|
|||||||
INTERVAL = int(os.environ.get("VORTEX_INTERVAL", "5"))
|
INTERVAL = int(os.environ.get("VORTEX_INTERVAL", "5"))
|
||||||
HOSTNAME = socket.gethostname()
|
HOSTNAME = socket.gethostname()
|
||||||
|
|
||||||
|
SESSION_IMAGE = os.environ.get("VORTEX_SESSION_IMAGE", "dorowu/ubuntu-desktop-lxde-vnc:latest")
|
||||||
HOME = os.path.expanduser("~")
|
HOME = os.path.expanduser("~")
|
||||||
COMFY_DIR = os.path.join(HOME, "ComfyUI")
|
|
||||||
COMFY_PY = os.path.join(COMFY_DIR, "venv", "bin", "python")
|
|
||||||
INST_DIR = os.path.join(HOME, "vortex-agent", "instances")
|
|
||||||
os.makedirs(INST_DIR, exist_ok=True)
|
|
||||||
|
|
||||||
_instance_pids = {} # instanceId -> pid
|
|
||||||
|
|
||||||
|
|
||||||
def http(method, path, body=None):
|
def http(method, path, body=None):
|
||||||
@@ -106,6 +98,10 @@ def uptime_sec():
|
|||||||
return int(float(open("/proc/uptime").read().split()[0]))
|
return int(float(open("/proc/uptime").read().split()[0]))
|
||||||
|
|
||||||
|
|
||||||
|
def _docker(args, timeout=180):
|
||||||
|
return subprocess.run(["docker"] + args, capture_output=True, text=True, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
def run_shell(command):
|
def run_shell(command):
|
||||||
try:
|
try:
|
||||||
r = subprocess.run(["bash", "-c", command], capture_output=True, text=True, timeout=600)
|
r = subprocess.run(["bash", "-c", command], capture_output=True, text=True, timeout=600)
|
||||||
@@ -114,46 +110,25 @@ def run_shell(command):
|
|||||||
return False, f"error: {e}"
|
return False, f"error: {e}"
|
||||||
|
|
||||||
|
|
||||||
def provision_comfyui(instance_id, port, user_dir):
|
def provision_ubuntu(instance_id, port, password, resolution):
|
||||||
os.makedirs(user_dir, exist_ok=True)
|
"""Spawn a full Ubuntu desktop session with the 4080 attached (noVNC on mapped port)."""
|
||||||
for sub in ("user", "output", "input", "models", "custom_nodes"):
|
name = f"vortex-{instance_id}"
|
||||||
os.makedirs(os.path.join(user_dir, sub), exist_ok=True)
|
_docker(["rm", "-f", name], timeout=30) # clean any stale instance
|
||||||
if not os.path.exists(os.path.join(COMFY_DIR, "main.py")):
|
r = _docker(["run", "-d", "--gpus", "all", "--name", name,
|
||||||
return False, "ComfyUI base not found on this node"
|
"-p", f"{port}:80",
|
||||||
if not os.path.exists(COMFY_PY):
|
"-e", f"VNC_PASSWORD={password}",
|
||||||
return False, "ComfyUI venv not found"
|
"-e", f"RESOLUTION={resolution or '1440x900'}",
|
||||||
args = [COMFY_PY, os.path.join(COMFY_DIR, "main.py"),
|
SESSION_IMAGE])
|
||||||
"--listen", "0.0.0.0", "--port", str(port),
|
if r.returncode == 0:
|
||||||
"--user-directory", os.path.join(user_dir, "user"),
|
cid = r.stdout.strip()[:12]
|
||||||
"--output-directory", os.path.join(user_dir, "output"),
|
return True, f"launched container={name} id={cid} port={port}"
|
||||||
"--input-directory", os.path.join(user_dir, "input")]
|
return False, f"failed: {r.stderr.strip()[:400]}"
|
||||||
try:
|
|
||||||
proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
||||||
start_new_session=True)
|
|
||||||
_instance_pids[instance_id] = proc.pid
|
|
||||||
with open(os.path.join(user_dir, "instance.pid"), "w") as f:
|
|
||||||
f.write(str(proc.pid))
|
|
||||||
return True, f"launched pid={proc.pid} port={port}"
|
|
||||||
except Exception as e:
|
|
||||||
return False, f"failed to launch: {e}"
|
|
||||||
|
|
||||||
|
|
||||||
def destroy_instance(instance_id):
|
def destroy_ubuntu(instance_id):
|
||||||
pid = _instance_pids.pop(instance_id, None)
|
name = f"vortex-{instance_id}"
|
||||||
if pid is None:
|
r = _docker(["rm", "-f", name], timeout=30)
|
||||||
pidfile = os.path.join(INST_DIR, instance_id, "instance.pid")
|
return True, (r.stdout.strip() or f"removed {name}")
|
||||||
if os.path.exists(pidfile):
|
|
||||||
try:
|
|
||||||
pid = int(open(pidfile).read().strip())
|
|
||||||
except Exception:
|
|
||||||
pid = None
|
|
||||||
if pid:
|
|
||||||
try:
|
|
||||||
os.kill(pid, 15)
|
|
||||||
return True, f"killed pid={pid}"
|
|
||||||
except Exception as e:
|
|
||||||
return True, f"kill pid={pid}: {e}"
|
|
||||||
return True, "no pid found"
|
|
||||||
|
|
||||||
|
|
||||||
def handle_job(job):
|
def handle_job(job):
|
||||||
@@ -162,14 +137,15 @@ def handle_job(job):
|
|||||||
payload = job.get("payload", {}) or {}
|
payload = job.get("payload", {}) or {}
|
||||||
if kind in ("shell", "hashcat", "comfyui"):
|
if kind in ("shell", "hashcat", "comfyui"):
|
||||||
ok, result = run_shell(cmd)
|
ok, result = run_shell(cmd)
|
||||||
elif kind == "provision_comfyui":
|
elif kind == "provision_ubuntu":
|
||||||
ok, result = provision_comfyui(payload.get("instanceId", "inst"),
|
ok, result = provision_ubuntu(payload.get("instanceId", "inst"),
|
||||||
payload.get("port", 8189),
|
int(payload.get("port", 6090)),
|
||||||
payload.get("userDir") or os.path.join(INST_DIR, payload.get("instanceId", "inst")))
|
payload.get("password", "vortex"),
|
||||||
elif kind == "destroy_instance":
|
payload.get("resolution", "1440x900"))
|
||||||
ok, result = destroy_instance(payload.get("instanceId", ""))
|
elif kind == "destroy_ubuntu":
|
||||||
|
ok, result = destroy_ubuntu(payload.get("instanceId", ""))
|
||||||
else:
|
else:
|
||||||
ok, result = False, "unknown job kind"
|
ok, result = False, f"unknown job kind: {kind}"
|
||||||
try:
|
try:
|
||||||
http("POST", f"/api/node/jobs/{job['id']}/result", {"ok": ok, "result": result})
|
http("POST", f"/api/node/jobs/{job['id']}/result", {"ok": ok, "result": result})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
Reference in New Issue
Block a user