tighten: ghost purge on VM removal, per-VM spawn error tolerance, DELETE device endpoint

app.py

@@ -285,6 +285,14 @@ def discovery_loop():
                     if row["ip"] and row["ip"] not in open_ips:
                         con.execute("UPDATE devices SET adb_state='offline' WHERE mac=?", (row["mac"],))
                         log("DEVICE_DISCONNECTED", f"device {row['mac']} went offline")
+                # purge ghosts: offline devices whose VM no longer exists in Proxmox
+                for row in con.execute("SELECT * FROM devices WHERE adb_state='offline' AND vmid IS NOT NULL").fetchall():
+                    r = pve("GET", f"/nodes/pve/qemu/{row['vmid']}/config") or {}
+                    if "does not exist" in str(r) or "no such VM" in str(r):
+                        con.execute("UPDATE proxies SET assigned_mac=NULL WHERE assigned_mac=?",
+                                    (row["mac"],))
+                        con.execute("DELETE FROM devices WHERE mac=?", (row["mac"],))
+                        log("GHOST_PURGED", f"device {row['mac'][:14]} (vmid {row['vmid']}) — VM gone, row removed")
             con.commit()
             con.close()
         except Exception as e:
@@ -547,9 +555,10 @@ def spawn_worker(job_id, count, prefix):
     con.commit()
     con.close()
     vmids = []
-    try:
-        for _ in range(count):
-            vmid = None
+    errors = []
+    for _ in range(count):
+        vmid = None
+        try:
             for cand in range(1310, 2000):
                 r = pve("GET", f"/nodes/pve/qemu/{cand}/status/current") or {}
                 if "does not exist" in str(r) or "no such VM" in str(r):
@@ -587,22 +596,21 @@ def spawn_worker(job_id, count, prefix):
             pve("POST", f"/nodes/pve/qemu/{vmid}/status/start", {})
             vmids.append(vmid)
             log("SPAWN", f"spawned {name} (vmid {vmid}, display {display})")
-            con = db()
-            con.execute("UPDATE spawn_jobs SET done=? WHERE id=?", (len(vmids), job_id))
-            con.commit()
-            con.close()
-            time.sleep(4)
+        except Exception as e:
+            errors.append(f"vmid {vmid}: {str(e)[:80]}")
+            log("SPAWN_ERROR", f"per-vm failure ({vmid}): {str(e)[:80]} — batch continues")
         con = db()
-        con.execute("UPDATE spawn_jobs SET status='done', vmid_list=? WHERE id=?", (json.dumps(vmids), job_id))
+        con.execute("UPDATE spawn_jobs SET done=? WHERE id=?", (len(vmids), job_id))
         con.commit()
         con.close()
-        log("SPAWN_DONE", f"job {job_id}: {len(vmids)} vms up")
-    except Exception as e:
-        con = db()
-        con.execute("UPDATE spawn_jobs SET status='error', error=? WHERE id=?", (str(e), job_id))
-        con.commit()
-        con.close()
-        log("SPAWN_ERROR", f"job {job_id}: {e}")
+        time.sleep(4)
+    con = db()
+    con.execute("UPDATE spawn_jobs SET status='done', vmid_list=?, error=? WHERE id=?",
+                (json.dumps(vmids), "; ".join(errors) or None, job_id))
+    con.commit()
+    con.close()
+    log("SPAWN_DONE", f"job {job_id}: {len(vmids)}/{count} vms up"
+        + (f" — errors: {'; '.join(errors)}" if errors else ""))
 
 @app.route("/api/spawn", methods=["POST"])
 def api_spawn():
@@ -683,6 +691,17 @@ def api_shot_latest(mac):
         return jsonify({"error": "no screenshot yet"}), 404
     return send_file(path, mimetype="image/png")
 
+@app.route("/api/device/<mac>", methods=["DELETE"])
+def api_delete_device(mac):
+    """Remove a device row (and release its proxy)."""
+    con = db()
+    con.execute("UPDATE proxies SET assigned_mac=NULL WHERE assigned_mac=?", (mac,))
+    con.execute("DELETE FROM devices WHERE mac=?", (mac,))
+    con.commit()
+    con.close()
+    log("DEVICE_DELETED", f"device {mac} removed manually")
+    return jsonify({"ok": True})
+
 @app.route("/api/device/<mac>/reboot", methods=["POST"])
 def api_reboot(mac):
     con = db()