first commit
This commit is contained in:
drjones
277
proxy_chain_manager/validator.py
Normal file
277
proxy_chain_manager/validator.py
Normal file
@@ -0,0 +1,277 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
import httpx
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Secondary fallback endpoints for IP resolution.
|
||||
# HTTP endpoints come first: many free proxies and older Windows Server
|
||||
# environments fail HTTPS CONNECT through chained hops. Plain HTTP IP-check
|
||||
# succeeds even when only forward-proxying (no CONNECT) works, so the chain's
|
||||
# "last leg" exit-IP verification doesn't fail just because TLS can't tunnel.
|
||||
_IP_FALLBACKS = [
|
||||
"http://api.ipify.org/?format=json",
|
||||
"http://checkip.amazonaws.com/",
|
||||
"http://ip-api.com/json/",
|
||||
"http://ifconfig.me/ip",
|
||||
"https://api.ipify.org?format=json",
|
||||
"https://httpbin.org/ip",
|
||||
"https://ifconfig.me/ip",
|
||||
]
|
||||
|
||||
|
||||
def _parse_ip(text: str) -> str | None:
|
||||
"""Parse an IP string from JSON or plain-text response body."""
|
||||
body = text.strip()
|
||||
if not body:
|
||||
return None
|
||||
if body.startswith("{"):
|
||||
try:
|
||||
d = json.loads(body)
|
||||
ip = d.get("ip") or d.get("origin") or d.get("query")
|
||||
return str(ip).split(",")[0].strip() if ip else None
|
||||
except Exception:
|
||||
return None
|
||||
# Plain-text: must look like an IP (short, no HTML)
|
||||
if len(body) < 50 and "<" not in body:
|
||||
candidate = body.split()[0].strip()
|
||||
if candidate.count(".") >= 3 or ":" in candidate:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
async def validate_proxies(
|
||||
proxy_urls: list[str],
|
||||
check_url: str,
|
||||
concurrency: int,
|
||||
timeout_seconds: float,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
target: int = 0,
|
||||
) -> list[str]:
|
||||
"""Validate proxies concurrently.
|
||||
|
||||
Args:
|
||||
target: Stop as soon as this many valid proxies are found (0 = test all).
|
||||
"""
|
||||
if not proxy_urls:
|
||||
return []
|
||||
sem = asyncio.Semaphore(max(1, concurrency))
|
||||
ok: list[str] = []
|
||||
lock = asyncio.Lock()
|
||||
total = len(proxy_urls)
|
||||
done = 0
|
||||
c = max(1, concurrency)
|
||||
waves = (total + c - 1) // c
|
||||
cap = min(600.0, float(waves) * float(timeout_seconds) + 45.0)
|
||||
cap = max(90.0, cap)
|
||||
log.info(
|
||||
"validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs target=%s check=%s",
|
||||
total,
|
||||
c,
|
||||
timeout_seconds,
|
||||
cap,
|
||||
target if target else "all",
|
||||
(check_url[:70] + "…") if len(check_url) > 70 else check_url,
|
||||
)
|
||||
|
||||
last_prog_t = 0.0
|
||||
prog_step = max(1, total // 120)
|
||||
enough = False # set True once target reached; tasks skip HTTP call and drain fast
|
||||
|
||||
async def one(u: str) -> None:
|
||||
nonlocal done, last_prog_t, enough
|
||||
# Skip cost of the HTTP check once we already have enough
|
||||
if enough:
|
||||
async with lock:
|
||||
done += 1
|
||||
if on_progress and done == total:
|
||||
on_progress(done, total)
|
||||
return
|
||||
async with sem:
|
||||
if enough: # re-check after potentially waiting on semaphore
|
||||
good = False
|
||||
else:
|
||||
good = await _check_one(u, check_url, timeout_seconds)
|
||||
async with lock:
|
||||
done += 1
|
||||
if good:
|
||||
ok.append(u)
|
||||
if target and len(ok) >= target:
|
||||
enough = True
|
||||
log.info("validate_proxies: target %d reached after %d tested", target, done)
|
||||
if on_progress:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
done == total
|
||||
or done == 1
|
||||
or enough
|
||||
or done % prog_step == 0
|
||||
or (now - last_prog_t) >= 0.1
|
||||
):
|
||||
last_prog_t = now
|
||||
on_progress(done, total)
|
||||
|
||||
async def _run_all() -> None:
|
||||
await asyncio.gather(*(one(u) for u in proxy_urls))
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(_run_all(), timeout=cap)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning(
|
||||
"validate_proxies wall-clock cap %.0fs exceeded (%d URLs) — returning partial results",
|
||||
cap,
|
||||
total,
|
||||
)
|
||||
log.info("validate_proxies: finished ok=%d / %d tested", len(ok), done)
|
||||
return ok
|
||||
|
||||
|
||||
async def _check_one(proxy_url: str, check_url: str, timeout_seconds: float) -> bool:
|
||||
"""Check a single proxy by fetching check_url through it. True = live and returns 200."""
|
||||
t = httpx.Timeout(timeout_seconds, connect=min(8.0, timeout_seconds))
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
proxy=proxy_url,
|
||||
timeout=t,
|
||||
verify=False,
|
||||
follow_redirects=True,
|
||||
) as c:
|
||||
r = await c.get(check_url)
|
||||
return r.status_code == 200 and len(r.content) > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def check_chain_exit_ip(
|
||||
listen_proxy: str,
|
||||
check_url: str,
|
||||
timeout_seconds: float,
|
||||
chain_hops: int = 3,
|
||||
) -> str | None:
|
||||
"""Query the IP-check URL through the local chain proxy.
|
||||
|
||||
Bounded total time — never stacks one slow request per fallback URL forever.
|
||||
Per-request timeout scales mildly with chain length so 5+ hop chains on
|
||||
slower hardware (older Windows Server, low-spec VPS) don't time out on
|
||||
cumulative TLS/CONNECT handshakes.
|
||||
"""
|
||||
hops = max(1, int(chain_hops))
|
||||
scale = 1.0 + max(0, hops - 3) * 0.35
|
||||
per = max(5.0, min(30.0, float(timeout_seconds) * scale))
|
||||
budget = max(20.0, min(90.0, float(timeout_seconds) * 2 * scale + 5.0))
|
||||
# Try more endpoints with HTTP first; if any hop blocks CONNECT, HTTPS
|
||||
# IP-check would fail and the whole chain looks "dead" at the last step.
|
||||
urls = [check_url] + [u for u in _IP_FALLBACKS if u != check_url][:4]
|
||||
log.debug(
|
||||
"check_chain_exit_ip: hops=%d budget=%.1fs per_req=%.1fs fallback_count=%d",
|
||||
hops,
|
||||
budget,
|
||||
per,
|
||||
len(urls),
|
||||
)
|
||||
|
||||
async def _run() -> str | None:
|
||||
t = httpx.Timeout(per, connect=min(8.0, per))
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
proxy=listen_proxy,
|
||||
timeout=t,
|
||||
verify=False,
|
||||
follow_redirects=True,
|
||||
) as c:
|
||||
for url in urls:
|
||||
try:
|
||||
r = await c.get(url)
|
||||
if r.status_code == 200:
|
||||
ip = _parse_ip(r.text)
|
||||
if ip:
|
||||
return ip
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_run(), timeout=budget)
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("check_chain_exit_ip timed out after %.1fs", budget)
|
||||
return None
|
||||
|
||||
|
||||
async def get_direct_ip(check_url: str, timeout_seconds: float = 15.0) -> str | None:
|
||||
"""Get our own exit IP without the proxy chain (so we can compare)."""
|
||||
per = max(5.0, min(15.0, float(timeout_seconds)))
|
||||
budget = max(12.0, min(45.0, float(timeout_seconds) * 2))
|
||||
urls = [check_url] + [u for u in _IP_FALLBACKS if u != check_url][:2]
|
||||
|
||||
async def _run() -> str | None:
|
||||
t = httpx.Timeout(per)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=t,
|
||||
verify=False,
|
||||
follow_redirects=True,
|
||||
) as c:
|
||||
for url in urls:
|
||||
try:
|
||||
r = await c.get(url)
|
||||
if r.status_code == 200:
|
||||
ip = _parse_ip(r.text)
|
||||
if ip:
|
||||
return ip
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_run(), timeout=budget)
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("get_direct_ip timed out after %.1fs", budget)
|
||||
return None
|
||||
|
||||
|
||||
async def check_https_tunnel(
|
||||
listen_proxy: str,
|
||||
timeout_seconds: float = 12.0,
|
||||
) -> tuple[bool, str]:
|
||||
"""Return (ok, message). True means the chain successfully tunneled HTTPS.
|
||||
|
||||
Many free proxies pass plain HTTP but refuse HTTP ``CONNECT`` (the method
|
||||
required to tunnel TLS). The chain can look healthy on an IP-check via
|
||||
plain HTTP yet break every HTTPS page a browser tries to load. This probe
|
||||
catches that case at start time so the user gets a clear warning instead
|
||||
of a mystery "browser broken".
|
||||
"""
|
||||
per = max(5.0, min(20.0, float(timeout_seconds)))
|
||||
t = httpx.Timeout(per, connect=min(8.0, per))
|
||||
# Use a tiny, fast, very-common HTTPS endpoint.
|
||||
targets = [
|
||||
"https://www.google.com/generate_204",
|
||||
"https://www.cloudflare.com/cdn-cgi/trace",
|
||||
"https://duckduckgo.com/",
|
||||
]
|
||||
last_err = ""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
proxy=listen_proxy, timeout=t, verify=False, follow_redirects=True,
|
||||
) as c:
|
||||
for url in targets:
|
||||
try:
|
||||
r = await c.get(url)
|
||||
if 200 <= r.status_code < 400:
|
||||
return True, f"HTTPS OK via {url} ({r.status_code})"
|
||||
except Exception as e:
|
||||
last_err = f"{type(e).__name__}: {e}"
|
||||
continue
|
||||
except Exception as e:
|
||||
last_err = f"{type(e).__name__}: {e}"
|
||||
return False, last_err or "all HTTPS targets failed"
|
||||
Reference in New Issue
Block a user