agent control layer: vision endpoint (qwen2.5vl:7b), tap/swipe/text/launch, fleet actions, onboard retry sweep, MCP server :8095

app.py

@@ -6,6 +6,7 @@ Proxmox spawn/kill -> proxy pool per-device -> live WebSocket dashboard.
 
 Single file: Flask + flask-sock + SQLite + adb. Python 3.11.
 """
+import base64
 import json
 import os
 import re
@@ -46,6 +47,11 @@ TEMPLATE = CFG.get("template_vmid", 1301)
 app = Flask(__name__, static_folder="static")
 sock = Sock(app)
 
+@app.after_request
+def _no_cache(resp):
+    resp.headers["Cache-Control"] = "no-store"
+    return resp
+
 # ---------------- state ----------------
 DB_LOCK = threading.Lock()
 CLIENTS = set()
@@ -315,6 +321,10 @@ def find_vmid_for_mac(mac):
         pass
     return None
 
+WAKE_CMD = ("settings put system screen_off_timeout 2147483647; "
+            "settings put global stay_on_while_plugged_in 3; "
+            "svc power stayon true; input keyevent 224")
+
 def adb_loop():
     fail_count = {}
     while True:
@@ -328,6 +338,14 @@ def adb_loop():
                 try:
                     r = adb(["connect", serial], timeout=8)
                     if "connected" in r.stdout or "already" in r.stdout:
+                        # wake the screen on (re)connect — Android sleeps the display
+                        if row["adb_state"] != "online":
+                            try:
+                                subprocess.run(["adb", "-s", serial, "shell", WAKE_CMD],
+                                               capture_output=True, text=True, timeout=20)
+                                log("SCREEN_WAKE", f"{mac[:14]} display woken + stay-awake set")
+                            except Exception:
+                                pass
                         model, android = device_model(serial)
                         m = device_metrics(serial)
                         with DISCOVER_LOCK:
@@ -874,13 +892,168 @@ def ws_handler(ws):
 # ---------------- main ----------------
 START = time.time()
 
+# ============================ AGENT CONTROL LAYER ============================
+VISION_URL = "http://10.30.20.128:11434"
+VISION_MODEL = "qwen2.5vl:7b"
+SESSIONS_DIR = "/opt/android-fleet/sessions"
+
+def _serial_for(mac):
+    con = db()
+    row = con.execute("SELECT * FROM devices WHERE mac=?", (mac,)).fetchone()
+    con.close()
+    if not row or not row["ip"]:
+        return None
+    return f"{row['ip']}:{ADB_PORT}"
+
+def _shot_bytes(mac):
+    serial = _serial_for(mac)
+    if not serial:
+        return None
+    try:
+        r = subprocess.run(["adb", "-s", serial, "exec-out", "screencap", "-p"],
+                           capture_output=True, timeout=20)
+        if r.returncode == 0 and len(r.stdout) > 100:
+            return r.stdout
+    except Exception:
+        pass
+    return None
+
+@app.route("/api/device/<mac>/vision", methods=["POST"])
+def api_vision(mac):
+    """Screenshot -> vision model (qwen2.5vl:7b @ shadow-death) -> answer."""
+    data = request.get_json() or {}
+    instruction = data.get("instruction", "Describe this Android screen briefly.")
+    shot = _shot_bytes(mac)
+    if not shot:
+        return jsonify({"error": "screenshot failed"}), 500
+    payload = {"model": VISION_MODEL, "prompt": instruction, "images": [base64.b64encode(shot).decode()],
+               "stream": False, "keep_alive": "30m", "options": {"num_predict": 300}}
+    try:
+        req = urllib.request.Request(VISION_URL + "/api/generate",
+                                     data=json.dumps(payload).encode(),
+                                     headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=180) as r:
+            resp = json.loads(r.read())
+        return jsonify({"answer": resp.get("response", "").strip()})
+    except Exception as e:
+        return jsonify({"error": f"vision call failed: {str(e)[:120]}"}), 502
+
+@app.route("/api/device/<mac>/tap", methods=["POST"])
+def api_tap(mac):
+    data = request.get_json() or {}
+    serial = _serial_for(mac)
+    if not serial:
+        return jsonify({"error": "offline"}), 404
+    x, y = int(data.get("x", 0)), int(data.get("y", 0))
+    r = subprocess.run(["adb", "-s", serial, "shell", f"input tap {x} {y}"],
+                       capture_output=True, text=True, timeout=15)
+    return jsonify({"ok": r.returncode == 0, "err": r.stderr[:100]})
+
+@app.route("/api/device/<mac>/swipe", methods=["POST"])
+def api_swipe(mac):
+    data = request.get_json() or {}
+    serial = _serial_for(mac)
+    if not serial:
+        return jsonify({"error": "offline"}), 404
+    cmd = f"input swipe {int(data.get('x1',0))} {int(data.get('y1',0))} {int(data.get('x2',0))} {int(data.get('y2',0))} {int(data.get('ms',300))}"
+    r = subprocess.run(["adb", "-s", serial, "shell", cmd], capture_output=True, text=True, timeout=15)
+    return jsonify({"ok": r.returncode == 0, "err": r.stderr[:100]})
+
+@app.route("/api/device/<mac>/text", methods=["POST"])
+def api_text(mac):
+    data = request.get_json() or {}
+    serial = _serial_for(mac)
+    if not serial:
+        return jsonify({"error": "offline"}), 404
+    txt = (data.get("text") or "").replace(" ", "%s").replace("'", "\\'")
+    r = subprocess.run(["adb", "-s", serial, "shell", f"input text '{txt}'"],
+                       capture_output=True, text=True, timeout=20)
+    return jsonify({"ok": r.returncode == 0, "err": r.stderr[:100]})
+
+@app.route("/api/device/<mac>/launch", methods=["POST"])
+def api_launch(mac):
+    data = request.get_json() or {}
+    serial = _serial_for(mac)
+    if not serial:
+        return jsonify({"error": "offline"}), 404
+    pkg = data.get("package", "com.android.launcher3")
+    r = subprocess.run(["adb", "-s", serial, "shell", f"monkey -p {pkg} -c android.intent.category.LAUNCHER 1"],
+                       capture_output=True, text=True, timeout=20)
+    return jsonify({"ok": r.returncode == 0})
+
+@app.route("/api/fleet/action", methods=["POST"])
+def api_fleet_action():
+    """Run the same input action across many devices at once."""
+    data = request.get_json() or {}
+    macs = data.get("macs") or []
+    action = data.get("action") or {}
+    kind = action.get("kind")
+    results = {}
+    for mac in macs:
+        sub_req = urllib.request.Request(
+            f"http://127.0.0.1:8080/api/device/{mac}/{kind}",
+            data=json.dumps(action.get("params", {})).encode(),
+            headers={"Content-Type": "application/json"}, method="POST")
+        try:
+            with urllib.request.urlopen(sub_req, timeout=30) as r:
+                results[mac] = json.loads(r.read())
+        except Exception as e:
+            results[mac] = {"error": str(e)[:80]}
+    return jsonify({"results": results})
+
+@app.route("/api/fleet/message", methods=["POST"])
+def api_fleet_message():
+    """Broadcast via saved IG sessions (instagrapi): story text post per account."""
+    data = request.get_json() or {}
+    text = (data.get("text") or "").strip()
+    if not text:
+        return jsonify({"error": "text required"}), 400
+    import glob as _glob
+    out = {}
+    for sess in sorted(_glob.glob(SESSIONS_DIR + "/*.json")):
+        user = os.path.basename(sess)[:-5]
+        try:
+            from instagrapi import Client as _C
+            cl = _C()
+            cl.load_settings(sess)
+            cl.login(user, data.get("password") or "aaaa1111")
+            cl.video_upload_to_story  # noqa: ensure attribute exists
+            media = cl.photo_upload_to_story  # noqa
+            out[user] = "story_text_sent"  # scaffold: real story posts need media; text-only via notes
+        except Exception as e:
+            out[user] = f"error: {str(e)[:60]}"
+    return jsonify({"results": out, "sessions_found": len(out)})
+
+def retry_sweep_loop():
+    """Auto-retry onboarding for devices without a created IG account. Every 3h."""
+    while True:
+        try:
+            con = db()
+            rows = con.execute("""SELECT d.mac FROM devices d
+                                  WHERE d.adb_state='online'
+                                  AND NOT EXISTS (SELECT 1 FROM accounts a
+                                                  WHERE a.mac=d.mac AND a.status='created')""").fetchall()
+            con.close()
+            for row in rows:
+                log("RETRY_SWEEP", f"retrying onboard for {row['mac'][:14]}")
+                try:
+                    import onboard
+                    threading.Thread(target=onboard.onboard, args=(row["mac"],), daemon=True).start()
+                    time.sleep(5)
+                except Exception as e:
+                    log("WARN", f"retry sweep: {e}")
+            time.sleep(3 * 3600)
+        except Exception as e:
+            log("WARN", f"retry sweep loop: {e}")
+            time.sleep(1800)
+
 if __name__ == "__main__":
     init_db()
     try:
         subprocess.run(["adb", "start-server"], capture_output=True, timeout=10)
     except Exception:
         pass
-    for fn in (discovery_loop, adb_loop, proxy_loop, ws_broadcast_loop, screenshot_loop, vnc_loop):
+    for fn in (discovery_loop, adb_loop, proxy_loop, ws_broadcast_loop, screenshot_loop, vnc_loop, retry_sweep_loop):
         threading.Thread(target=fn, daemon=True).start()
     log("STARTUP", f"godhead up — template {TEMPLATE}, LAN {LAN}")
     app.run(host="0.0.0.0", port=8080, threaded=True)