first commit

proxy_chain_manager/fetcher.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+log = logging.getLogger(__name__)
+
+
+def fetch_proxy_json(url: str, timeout: float = 45.0) -> list[dict[str, Any]]:
+    log.debug("fetch_proxy_json: GET %s", (url[:100] + "…") if len(url) > 100 else url)
+    with httpx.Client(timeout=timeout, follow_redirects=True) as c:
+        r = c.get(url)
+        r.raise_for_status()
+        data = r.json()
+    if not isinstance(data, list):
+        return []
+    return [x for x in data if isinstance(x, dict)]
+
+
+def normalize_entries(rows: list[dict[str, Any]], prefer_elite: bool) -> list[str]:
+    out: list[str] = []
+    for row in rows:
+        if prefer_elite and str(row.get("anonymity", "")).lower() != "elite":
+            continue
+        p = row.get("proxy")
+        if isinstance(p, str) and "://" in p:
+            out.append(p.strip())
+    # de-dupe preserving order
+    seen: set[str] = set()
+    uniq: list[str] = []
+    for u in out:
+        if u not in seen:
+            seen.add(u)
+            uniq.append(u)
+    return uniq