From 476ab669ee00fa90eca986de4bdabd0e5f595f99 Mon Sep 17 00:00:00 2001 From: drjones Date: Thu, 13 Aug 2026 19:29:30 -0700 Subject: [PATCH] google account flow driver: vision-driven webview walkthrough with tolerant action parser --- scripts/google_flow.py | 149 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 scripts/google_flow.py diff --git a/scripts/google_flow.py b/scripts/google_flow.py new file mode 100644 index 0000000..0dda731 --- /dev/null +++ b/scripts/google_flow.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Google create-account walkthrough v3 — vision-driven for the WEBVIEW flow. +Each step: ask qwen2.5vl for one specific element's coordinates, act, log, shoot. +1024x768 screen. Example data only — stops at whatever wall Google puts up. +""" +import json, os, re, sys, time, urllib.request + +BASE = "http://10.30.20.31:8080" +TOKEN = "godhead-fleet-2026" +SHOTS = "/tmp/flow_shots" + +def api(path, method="GET", payload=None, timeout=130): + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(BASE + path, data=data, method=method, + headers={"Content-Type": "application/json", + "X-Auth-Token": TOKEN}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read()) + +def vision(mac, instruction): + try: + return api(f"/api/device/{mac}/vision", "POST", {"instruction": instruction}, 130).get("answer", "") + except Exception as e: + return f"VISION_ERR {e}" + +def tap(mac, x, y): + return api(f"/api/device/{mac}/tap", "POST", {"x": int(x), "y": int(y)}, 30) + +def type_text(mac, text): + return api(f"/api/device/{mac}/text", "POST", {"text": text}, 40) + +def shell(mac, cmd): + try: + return api(f"/api/device/{mac}/exec", "POST", {"cmd": cmd}, 60).get("output", "") + except Exception: + return "" + +def shot(mac, tag): + os.makedirs(SHOTS, exist_ok=True) + p = f"{SHOTS}/{mac[-4:]}_{tag}.png" + try: + req = urllib.request.Request(BASE + f"/api/device/{mac}/screenshot/latest?t={int(time.time())}") + with urllib.request.urlopen(req, timeout=40) as r: + open(p, "wb").write(r.read()) + except Exception: + pass + return p + +def parse_action(ans): + """Tolerant parser for vision-model action output (many JSON flavors).""" + if '"none"' in ans or "'none'" in ans: + m = re.search(r'note"?\s*[:=]\s*"([^"]+)"', ans) + return ("none", m.group(1) if m else "unspecified") + # 1) explicit x/y pairs in any order, quotes optional + for rx in (r'["\']?x["\']?\s*[:=]\s*["\']?(\d+)["\']?[^0-9]{0,20}["\']?y["\']?\s*[:=]\s*["\']?(\d+)["\']?', + r'coordinate["\']?\s*[:=]\s*\[\s*(\d+)\s*,\s*(\d+)\s*\]', + r'\((\d{2,4})\s*,\s*(\d{2,4})\)'): + m = re.search(rx, ans) + if m: + x, y = int(m.group(1)), int(m.group(2)) + if 0 <= x <= 1100 and 0 <= y <= 800: + return ("tap", x, y) + # 2) number pairs anywhere (last resort): find the first two 3-digit ints near "point" + m = re.search(r'point[^0-9]{0,15}(\d{2,4})[^0-9]{0,15}(\d{2,4})', ans) + if m: + return ("tap", int(m.group(1)), int(m.group(2))) + return None + +def ask_tap(mac, prompt, tag, tries=2): + """Ask vision where to tap, tap it. Returns (ok, detail).""" + instr = (prompt + " The screen is 1024x768. Reply with ONLY JSON: " + "{\"action\":\"tap\",\"x\":INT,\"y\":INT} or {\"action\":\"none\",\"note\":\"why\"}.") + last = "" + for i in range(tries): + ans = vision(mac, instr) + last = ans + act = parse_action(ans) + if act and act[0] == "tap": + r = tap(mac, act[1], act[2]) + shot(mac, tag) + return True, f"tapped {act[1]},{act[2]} | vision: {ans[:80]}" + if act and act[0] == "none": + return False, f"vision said none: {act[1][:80]} | {ans[:120]}" + time.sleep(3) + return False, f"no coords: {last[:150]}" + +def ask_fill(mac, field_prompt, value, tag): + """Vision locates the text field, tap it, type value.""" + instr = (field_prompt + " The screen is 1024x768. Reply with ONLY JSON: " + "{\"action\":\"tap\",\"x\":INT,\"y\":INT} pointing at the CENTER of that text field.") + ans = vision(mac, instr) + act = parse_action(ans) + if act and act[0] == "tap": + tap(mac, act[1], act[2]) + time.sleep(2) + shell(mac, "input keyevent 123") + type_text(mac, value) + shot(mac, tag) + return True, f"filled '{value}' at {act[1]},{act[2]}" + return False, f"field not found: {ans[:120]}" + +def run(mac, counter): + log = [] + def s(tag, ok, msg): + log.append(f"[{tag}] {msg}") + print(f"[{tag}] {msg}", flush=True) + + # ensure we're at the Google sign-in screen + shell(mac, "am start -a android.settings.ADD_ACCOUNT_SETTINGS") + time.sleep(4) + shell(mac, "input tap 217 223") # the 'Google' row (worked on 1310) + time.sleep(6) + + s("1_create", *ask_tap(mac, "Find the link or button labeled 'Create account' and give its center coordinates.", "create")) + time.sleep(6) + s("2_myself", *ask_tap(mac, "Find the option 'For myself' (not 'For my child') and give its center coordinates.", "myself")) + time.sleep(5) + s("3_first", *ask_fill(mac, "Find the 'First name' text field.", "Alex", "first")) + s("3_last", *ask_fill(mac, "Find the 'Last name' text field.", "Smith", "last")) + time.sleep(2) + s("4_next", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next1")) + time.sleep(6) + # birthday: try Next with defaults first + s("5_bday", *ask_tap(mac, "This is the birthday screen. Find the blue 'Next' button (or 'Month' dropdown if Next is greyed) and give its center coordinates.", "bday")) + time.sleep(6) + s("6_gender", *ask_tap(mac, "On the gender screen find 'Rather not say' OR the blue 'Next' button — give the center of whichever exists.", "gender")) + time.sleep(5) + s("7_user", *ask_fill(mac, "Find the 'Username' text field (the create-your-gmail-address field).", f"alexsmith{counter}", "user")) + time.sleep(3) + s("8_next2", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next2")) + time.sleep(6) + pw = "J0nes123!xYz" + s("9_pw", *ask_fill(mac, "Find the 'Password' field (first of the two password fields).", pw, "pw")) + s("9_pw2", *ask_fill(mac, "Find the 'Confirm' password field (the second one).", pw, "pw2")) + time.sleep(3) + s("10_next3", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next3")) + time.sleep(8) + # capture the final screen + ans = vision(mac, "Quote all visible text on this screen. Is there a phone number requirement, a CAPTCHA, an error, or a 'verify' screen?") + s("FINAL", False, ans[:250]) + shot(mac, "final") + return log + +if __name__ == "__main__": + mac = sys.argv[1] + counter = sys.argv[2] if len(sys.argv) > 2 else "301" + print(f"=== Google create-account walkthrough on {mac} (alexsmith{counter}) ===", flush=True) + run(mac, counter)