google account flow driver: vision-driven webview walkthrough with tolerant action parser
This commit is contained in:
drjones
149
scripts/google_flow.py
Normal file
149
scripts/google_flow.py
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Google create-account walkthrough v3 — vision-driven for the WEBVIEW flow.
|
||||||
|
Each step: ask qwen2.5vl for one specific element's coordinates, act, log, shoot.
|
||||||
|
1024x768 screen. Example data only — stops at whatever wall Google puts up.
|
||||||
|
"""
|
||||||
|
import json, os, re, sys, time, urllib.request
|
||||||
|
|
||||||
|
BASE = "http://10.30.20.31:8080"
|
||||||
|
TOKEN = "godhead-fleet-2026"
|
||||||
|
SHOTS = "/tmp/flow_shots"
|
||||||
|
|
||||||
|
def api(path, method="GET", payload=None, timeout=130):
|
||||||
|
data = json.dumps(payload).encode() if payload is not None else None
|
||||||
|
req = urllib.request.Request(BASE + path, data=data, method=method,
|
||||||
|
headers={"Content-Type": "application/json",
|
||||||
|
"X-Auth-Token": TOKEN})
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return json.loads(r.read())
|
||||||
|
|
||||||
|
def vision(mac, instruction):
|
||||||
|
try:
|
||||||
|
return api(f"/api/device/{mac}/vision", "POST", {"instruction": instruction}, 130).get("answer", "")
|
||||||
|
except Exception as e:
|
||||||
|
return f"VISION_ERR {e}"
|
||||||
|
|
||||||
|
def tap(mac, x, y):
|
||||||
|
return api(f"/api/device/{mac}/tap", "POST", {"x": int(x), "y": int(y)}, 30)
|
||||||
|
|
||||||
|
def type_text(mac, text):
|
||||||
|
return api(f"/api/device/{mac}/text", "POST", {"text": text}, 40)
|
||||||
|
|
||||||
|
def shell(mac, cmd):
|
||||||
|
try:
|
||||||
|
return api(f"/api/device/{mac}/exec", "POST", {"cmd": cmd}, 60).get("output", "")
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def shot(mac, tag):
|
||||||
|
os.makedirs(SHOTS, exist_ok=True)
|
||||||
|
p = f"{SHOTS}/{mac[-4:]}_{tag}.png"
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(BASE + f"/api/device/{mac}/screenshot/latest?t={int(time.time())}")
|
||||||
|
with urllib.request.urlopen(req, timeout=40) as r:
|
||||||
|
open(p, "wb").write(r.read())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return p
|
||||||
|
|
||||||
|
def parse_action(ans):
|
||||||
|
"""Tolerant parser for vision-model action output (many JSON flavors)."""
|
||||||
|
if '"none"' in ans or "'none'" in ans:
|
||||||
|
m = re.search(r'note"?\s*[:=]\s*"([^"]+)"', ans)
|
||||||
|
return ("none", m.group(1) if m else "unspecified")
|
||||||
|
# 1) explicit x/y pairs in any order, quotes optional
|
||||||
|
for rx in (r'["\']?x["\']?\s*[:=]\s*["\']?(\d+)["\']?[^0-9]{0,20}["\']?y["\']?\s*[:=]\s*["\']?(\d+)["\']?',
|
||||||
|
r'coordinate["\']?\s*[:=]\s*\[\s*(\d+)\s*,\s*(\d+)\s*\]',
|
||||||
|
r'\((\d{2,4})\s*,\s*(\d{2,4})\)'):
|
||||||
|
m = re.search(rx, ans)
|
||||||
|
if m:
|
||||||
|
x, y = int(m.group(1)), int(m.group(2))
|
||||||
|
if 0 <= x <= 1100 and 0 <= y <= 800:
|
||||||
|
return ("tap", x, y)
|
||||||
|
# 2) number pairs anywhere (last resort): find the first two 3-digit ints near "point"
|
||||||
|
m = re.search(r'point[^0-9]{0,15}(\d{2,4})[^0-9]{0,15}(\d{2,4})', ans)
|
||||||
|
if m:
|
||||||
|
return ("tap", int(m.group(1)), int(m.group(2)))
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ask_tap(mac, prompt, tag, tries=2):
|
||||||
|
"""Ask vision where to tap, tap it. Returns (ok, detail)."""
|
||||||
|
instr = (prompt + " The screen is 1024x768. Reply with ONLY JSON: "
|
||||||
|
"{\"action\":\"tap\",\"x\":INT,\"y\":INT} or {\"action\":\"none\",\"note\":\"why\"}.")
|
||||||
|
last = ""
|
||||||
|
for i in range(tries):
|
||||||
|
ans = vision(mac, instr)
|
||||||
|
last = ans
|
||||||
|
act = parse_action(ans)
|
||||||
|
if act and act[0] == "tap":
|
||||||
|
r = tap(mac, act[1], act[2])
|
||||||
|
shot(mac, tag)
|
||||||
|
return True, f"tapped {act[1]},{act[2]} | vision: {ans[:80]}"
|
||||||
|
if act and act[0] == "none":
|
||||||
|
return False, f"vision said none: {act[1][:80]} | {ans[:120]}"
|
||||||
|
time.sleep(3)
|
||||||
|
return False, f"no coords: {last[:150]}"
|
||||||
|
|
||||||
|
def ask_fill(mac, field_prompt, value, tag):
|
||||||
|
"""Vision locates the text field, tap it, type value."""
|
||||||
|
instr = (field_prompt + " The screen is 1024x768. Reply with ONLY JSON: "
|
||||||
|
"{\"action\":\"tap\",\"x\":INT,\"y\":INT} pointing at the CENTER of that text field.")
|
||||||
|
ans = vision(mac, instr)
|
||||||
|
act = parse_action(ans)
|
||||||
|
if act and act[0] == "tap":
|
||||||
|
tap(mac, act[1], act[2])
|
||||||
|
time.sleep(2)
|
||||||
|
shell(mac, "input keyevent 123")
|
||||||
|
type_text(mac, value)
|
||||||
|
shot(mac, tag)
|
||||||
|
return True, f"filled '{value}' at {act[1]},{act[2]}"
|
||||||
|
return False, f"field not found: {ans[:120]}"
|
||||||
|
|
||||||
|
def run(mac, counter):
|
||||||
|
log = []
|
||||||
|
def s(tag, ok, msg):
|
||||||
|
log.append(f"[{tag}] {msg}")
|
||||||
|
print(f"[{tag}] {msg}", flush=True)
|
||||||
|
|
||||||
|
# ensure we're at the Google sign-in screen
|
||||||
|
shell(mac, "am start -a android.settings.ADD_ACCOUNT_SETTINGS")
|
||||||
|
time.sleep(4)
|
||||||
|
shell(mac, "input tap 217 223") # the 'Google' row (worked on 1310)
|
||||||
|
time.sleep(6)
|
||||||
|
|
||||||
|
s("1_create", *ask_tap(mac, "Find the link or button labeled 'Create account' and give its center coordinates.", "create"))
|
||||||
|
time.sleep(6)
|
||||||
|
s("2_myself", *ask_tap(mac, "Find the option 'For myself' (not 'For my child') and give its center coordinates.", "myself"))
|
||||||
|
time.sleep(5)
|
||||||
|
s("3_first", *ask_fill(mac, "Find the 'First name' text field.", "Alex", "first"))
|
||||||
|
s("3_last", *ask_fill(mac, "Find the 'Last name' text field.", "Smith", "last"))
|
||||||
|
time.sleep(2)
|
||||||
|
s("4_next", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next1"))
|
||||||
|
time.sleep(6)
|
||||||
|
# birthday: try Next with defaults first
|
||||||
|
s("5_bday", *ask_tap(mac, "This is the birthday screen. Find the blue 'Next' button (or 'Month' dropdown if Next is greyed) and give its center coordinates.", "bday"))
|
||||||
|
time.sleep(6)
|
||||||
|
s("6_gender", *ask_tap(mac, "On the gender screen find 'Rather not say' OR the blue 'Next' button — give the center of whichever exists.", "gender"))
|
||||||
|
time.sleep(5)
|
||||||
|
s("7_user", *ask_fill(mac, "Find the 'Username' text field (the create-your-gmail-address field).", f"alexsmith{counter}", "user"))
|
||||||
|
time.sleep(3)
|
||||||
|
s("8_next2", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next2"))
|
||||||
|
time.sleep(6)
|
||||||
|
pw = "J0nes123!xYz"
|
||||||
|
s("9_pw", *ask_fill(mac, "Find the 'Password' field (first of the two password fields).", pw, "pw"))
|
||||||
|
s("9_pw2", *ask_fill(mac, "Find the 'Confirm' password field (the second one).", pw, "pw2"))
|
||||||
|
time.sleep(3)
|
||||||
|
s("10_next3", *ask_tap(mac, "Find the blue 'Next' button and give its center coordinates.", "next3"))
|
||||||
|
time.sleep(8)
|
||||||
|
# capture the final screen
|
||||||
|
ans = vision(mac, "Quote all visible text on this screen. Is there a phone number requirement, a CAPTCHA, an error, or a 'verify' screen?")
|
||||||
|
s("FINAL", False, ans[:250])
|
||||||
|
shot(mac, "final")
|
||||||
|
return log
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
mac = sys.argv[1]
|
||||||
|
counter = sys.argv[2] if len(sys.argv) > 2 else "301"
|
||||||
|
print(f"=== Google create-account walkthrough on {mac} (alexsmith{counter}) ===", flush=True)
|
||||||
|
run(mac, counter)
|
||||||
Reference in New Issue
Block a user