#!/usr/bin/env python3 """ Google create-account walkthrough v3 — vision-driven for the WEBVIEW flow. Each step: ask qwen2.5vl for one specific element's coordinates, act, log, shoot. 1024x768 screen. Example data only — stops at whatever wall Google puts up. """ import json, os, re, sys, time, urllib.request BASE = "http://10.30.20.31:8080" TOKEN = "godhead-fleet-2026" SHOTS = "/tmp/flow_shots" def api(path, method="GET", payload=None, timeout=130): data = json.dumps(payload).encode() if payload is not None else None req = urllib.request.Request(BASE + path, data=data, method=method, headers={"Content-Type": "application/json", "X-Auth-Token": TOKEN}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read()) def vision(mac, instruction): try: return api(f"/api/device/{mac}/vision", "POST", {"instruction": instruction}, 130).get("answer", "") except Exception as e: return f"VISION_ERR {e}" def tap(mac, x, y): return api(f"/api/device/{mac}/tap", "POST", {"x": int(x), "y": int(y)}, 30) def type_text(mac, text): return api(f"/api/device/{mac}/text", "POST", {"text": text}, 40) def shell(mac, cmd): try: return api(f"/api/device/{mac}/exec", "POST", {"cmd": cmd}, 60).get("output", "") except Exception: return "" def shot(mac, tag): os.makedirs(SHOTS, exist_ok=True) p = f"{SHOTS}/{mac[-4:]}_{tag}.png" try: req = urllib.request.Request(BASE + f"/api/device/{mac}/screenshot/latest?t={int(time.time())}") with urllib.request.urlopen(req, timeout=40) as r: open(p, "wb").write(r.read()) except Exception: pass return p def parse_action(ans): """Tolerant parser for vision-model action output (many JSON flavors).""" if '"none"' in ans or "'none'" in ans: m = re.search(r'note"?\s*[:=]\s*"([^"]+)"', ans) return ("none", m.group(1) if m else "unspecified") # 1) explicit x/y pairs in any order, quotes optional for rx in (r'["\']?x["\']?\s*[:=]\s*["\']?(\d+)["\']?[^0-9]{0,20}["\']?y["\']?\s*[:=]\s*["\']?(\d+)["\']?', r'coordinate["\']?\s*[:=]\s*\[\s*(\d+)\s*,\s*(\d+)\s*\]', r'\((\d{2,4})\s*,\s*(\d{2,4})\)'): m = re.search(rx, ans) if m: x, y = int(m.group(1)), int(m.group(2)) if 0 <= x <= 1100 and 0 <= y <= 800: return ("tap", x, y) # 2) number pairs anywhere (last resort): find the first two 3-digit ints near "point" m = re.search(r'point[^0-9]{0,15}(\d{2,4})[^0-9]{0,15}(\d{2,4})', ans) if m: return ("tap", int(m.group(1)), int(m.group(2))) return None def ask_tap(mac, prompt, tag, tries=2): """Ask vision where to tap, tap it. Returns (ok, detail).""" instr = (prompt + " The screen is 1024x768. Reply with ONLY JSON: " "{\"action\":\"tap\",\"x\":INT,\"y\":INT} or {\"action\":\"none\",\"note\":\"why\"}.") last = "" for i in range(tries): ans = vision(mac, instr) last = ans act = parse_action(ans) if act and act[0] == "tap": r = tap(mac, act[1], act[2]) shot(mac, tag) return True, f"tapped {act[1]},{act[2]} | vision: {ans[:80]}" if act and act[0] == "none": return False, f"vision said none: {act[1][:80]} | {ans[:120]}" time.sleep(3) return False, f"no coords: {last[:150]}" def ask_fill(mac, field_prompt, value, tag): """Vision locates the text field, tap it, type value.""" instr = (field_prompt + " The screen is 1024x768. Reply with ONLY JSON: " "{\"action\":\"tap\",\"x\":INT,\"y\":INT} pointing at the CENTER of that text field.") ans = vision(mac, instr) act = parse_action(ans) if act and act[0] == "tap": tap(mac, act[1], act[2]) time.sleep(2) shell(mac, "input keyevent 123") type_text(mac, value) shot(mac, tag) return True, f"filled '{value}' at {act[1]},{act[2]}" return False, f"field not found: {ans[:120]}" def run(mac, counter): log = [] def s(tag, ok, msg): log.append(f"[{tag}] {msg}") print(f"[{tag}] {msg}", flush=True) # ensure we're at the Google sign-in screen shell(mac, "am start -a android.settings.ADD_ACCOUNT_SETTINGS") time.sleep(4) shell(mac, "input tap 217 223") # the 'Google' row (worked on 1310) time.sleep(6) s("1_create", *ask_tap(mac, "Find the link or button labeled 'Create account' and give its center coordinates.", "create")) time.sleep(6) s("2_myself", *ask_tap(mac, "Find the option 'For myself' (not 'For my child') and give its center coordinates.", "myself")) time.sleep(5) s("3_first", *ask_fill(mac, "Find the 'First name' text field.", "Alex", "first")) s("3_last", *ask_fill(mac, "Find the 'Last name' text field.", "Smith", "last")) time.sleep(2) s("4_next", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next1")) time.sleep(6) # birthday: try Next with defaults first s("5_bday", *ask_tap(mac, "This is the birthday screen. Find the blue 'Next' button (or 'Month' dropdown if Next is greyed) and give its center coordinates.", "bday")) time.sleep(6) s("6_gender", *ask_tap(mac, "On the gender screen find 'Rather not say' OR the blue 'Next' button — give the center of whichever exists.", "gender")) time.sleep(5) s("7_user", *ask_fill(mac, "Find the 'Username' text field (the create-your-gmail-address field).", f"alexsmith{counter}", "user")) time.sleep(3) s("8_next2", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next2")) time.sleep(6) pw = "J0nes123!xYz" s("9_pw", *ask_fill(mac, "Find the 'Password' field (first of the two password fields).", pw, "pw")) s("9_pw2", *ask_fill(mac, "Find the 'Confirm' password field (the second one).", pw, "pw2")) time.sleep(3) s("10_next3", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next3")) time.sleep(8) # capture the final screen ans = vision(mac, "Quote all visible text on this screen. Is there a phone number requirement, a CAPTCHA, an error, or a 'verify' screen?") s("FINAL", False, ans[:250]) shot(mac, "final") return log if __name__ == "__main__": mac = sys.argv[1] counter = sys.argv[2] if len(sys.argv) > 2 else "301" print(f"=== Google create-account walkthrough on {mac} (alexsmith{counter}) ===", flush=True) run(mac, counter)