37 lines
986 B
Python
37 lines
986 B
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def fetch_proxy_json(url: str, timeout: float = 45.0) -> list[dict[str, Any]]:
|
|
with httpx.Client(timeout=timeout, follow_redirects=True) as c:
|
|
r = c.get(url)
|
|
r.raise_for_status()
|
|
data = r.json()
|
|
if not isinstance(data, list):
|
|
return []
|
|
return [x for x in data if isinstance(x, dict)]
|
|
|
|
|
|
def normalize_entries(rows: list[dict[str, Any]], prefer_elite: bool) -> list[str]:
|
|
out: list[str] = []
|
|
for row in rows:
|
|
if prefer_elite and str(row.get("anonymity", "")).lower() != "elite":
|
|
continue
|
|
p = row.get("proxy")
|
|
if isinstance(p, str) and "://" in p:
|
|
out.append(p.strip())
|
|
# de-dupe preserving order
|
|
seen: set[str] = set()
|
|
uniq: list[str] = []
|
|
for u in out:
|
|
if u not in seen:
|
|
seen.add(u)
|
|
uniq.append(u)
|
|
return uniq
|