proxy-god/proxy_chain_manager/validator.py

from __future__ import annotations

import asyncio
import json
import logging
import time
from typing import Callable

import httpx

log = logging.getLogger(__name__)

# Secondary fallback endpoints for IP resolution.
# HTTP endpoints come first: many free proxies and older Windows Server
# environments fail HTTPS CONNECT through chained hops. Plain HTTP IP-check
# succeeds even when only forward-proxying (no CONNECT) works, so the chain's
# "last leg" exit-IP verification doesn't fail just because TLS can't tunnel.
_IP_FALLBACKS = [
    "http://api.ipify.org/?format=json",
    "http://checkip.amazonaws.com/",
    "http://ip-api.com/json/",
    "http://ifconfig.me/ip",
    "https://api.ipify.org?format=json",
    "https://httpbin.org/ip",
    "https://ifconfig.me/ip",
]


def _parse_ip(text: str) -> str | None:
    """Parse an IP string from JSON or plain-text response body."""
    body = text.strip()
    if not body:
        return None
    if body.startswith("{"):
        try:
            d = json.loads(body)
            ip = d.get("ip") or d.get("origin") or d.get("query")
            return str(ip).split(",")[0].strip() if ip else None
        except Exception:
            return None
    # Plain-text: must look like an IP (short, no HTML)
    if len(body) < 50 and "<" not in body:
        candidate = body.split()[0].strip()
        if candidate.count(".") >= 3 or ":" in candidate:
            return candidate
    return None


async def validate_proxies(
    proxy_urls: list[str],
    check_url: str,
    concurrency: int,
    timeout_seconds: float,
    on_progress: Callable[[int, int], None] | None = None,
    target: int = 0,
) -> list[str]:
    """Validate proxies concurrently.

    Args:
        target: Stop as soon as this many valid proxies are found (0 = test all).
    """
    if not proxy_urls:
        return []
    sem = asyncio.Semaphore(max(1, concurrency))
    ok: list[str] = []
    lock = asyncio.Lock()
    total = len(proxy_urls)
    done = 0
    c = max(1, concurrency)
    waves = (total + c - 1) // c
    cap = min(600.0, float(waves) * float(timeout_seconds) + 45.0)
    cap = max(90.0, cap)
    log.info(
        "validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs target=%s check=%s",
        total,
        c,
        timeout_seconds,
        cap,
        target if target else "all",
        (check_url[:70] + "…") if len(check_url) > 70 else check_url,
    )

    last_prog_t = 0.0
    prog_step = max(1, total // 120)
    enough = False   # set True once target reached; tasks skip HTTP call and drain fast

    async def one(u: str) -> None:
        nonlocal done, last_prog_t, enough
        # Skip cost of the HTTP check once we already have enough
        if enough:
            async with lock:
                done += 1
                if on_progress and done == total:
                    on_progress(done, total)
            return
        async with sem:
            if enough:   # re-check after potentially waiting on semaphore
                good = False
            else:
                good = await _check_one(u, check_url, timeout_seconds)
        async with lock:
            done += 1
            if good:
                ok.append(u)
                if target and len(ok) >= target:
                    enough = True
                    log.info("validate_proxies: target %d reached after %d tested", target, done)
            if on_progress:
                now = time.monotonic()
                if (
                    done == total
                    or done == 1
                    or enough
                    or done % prog_step == 0
                    or (now - last_prog_t) >= 0.1
                ):
                    last_prog_t = now
                    on_progress(done, total)

    async def _run_all() -> None:
        await asyncio.gather(*(one(u) for u in proxy_urls))

    try:
        await asyncio.wait_for(_run_all(), timeout=cap)
    except asyncio.TimeoutError:
        log.warning(
            "validate_proxies wall-clock cap %.0fs exceeded (%d URLs) — returning partial results",
            cap,
            total,
        )
    log.info("validate_proxies: finished ok=%d / %d tested", len(ok), done)
    return ok


async def _check_one(proxy_url: str, check_url: str, timeout_seconds: float) -> bool:
    """Check a single proxy by fetching check_url through it. True = live and returns 200."""
    t = httpx.Timeout(timeout_seconds, connect=min(8.0, timeout_seconds))
    try:
        async with httpx.AsyncClient(
            proxy=proxy_url,
            timeout=t,
            verify=False,
            follow_redirects=True,
        ) as c:
            r = await c.get(check_url)
            return r.status_code == 200 and len(r.content) > 0
    except Exception:
        return False


async def check_chain_exit_ip(
    listen_proxy: str,
    check_url: str,
    timeout_seconds: float,
    chain_hops: int = 3,
) -> str | None:
    """Query the IP-check URL through the local chain proxy.

    Bounded total time — never stacks one slow request per fallback URL forever.
    Per-request timeout scales mildly with chain length so 5+ hop chains on
    slower hardware (older Windows Server, low-spec VPS) don't time out on
    cumulative TLS/CONNECT handshakes.
    """
    hops = max(1, int(chain_hops))
    scale = 1.0 + max(0, hops - 3) * 0.35
    per = max(5.0, min(30.0, float(timeout_seconds) * scale))
    budget = max(20.0, min(90.0, float(timeout_seconds) * 2 * scale + 5.0))
    # Try more endpoints with HTTP first; if any hop blocks CONNECT, HTTPS
    # IP-check would fail and the whole chain looks "dead" at the last step.
    urls = [check_url] + [u for u in _IP_FALLBACKS if u != check_url][:4]
    log.debug(
        "check_chain_exit_ip: hops=%d budget=%.1fs per_req=%.1fs fallback_count=%d",
        hops,
        budget,
        per,
        len(urls),
    )

    async def _run() -> str | None:
        t = httpx.Timeout(per, connect=min(8.0, per))
        try:
            async with httpx.AsyncClient(
                proxy=listen_proxy,
                timeout=t,
                verify=False,
                follow_redirects=True,
            ) as c:
                for url in urls:
                    try:
                        r = await c.get(url)
                        if r.status_code == 200:
                            ip = _parse_ip(r.text)
                            if ip:
                                return ip
                    except Exception:
                        continue
        except Exception:
            pass
        return None

    try:
        return await asyncio.wait_for(_run(), timeout=budget)
    except asyncio.TimeoutError:
        log.debug("check_chain_exit_ip timed out after %.1fs", budget)
        return None


async def get_direct_ip(check_url: str, timeout_seconds: float = 15.0) -> str | None:
    """Get our own exit IP without the proxy chain (so we can compare)."""
    per = max(5.0, min(15.0, float(timeout_seconds)))
    budget = max(12.0, min(45.0, float(timeout_seconds) * 2))
    urls = [check_url] + [u for u in _IP_FALLBACKS if u != check_url][:2]

    async def _run() -> str | None:
        t = httpx.Timeout(per)
        try:
            async with httpx.AsyncClient(
                timeout=t,
                verify=False,
                follow_redirects=True,
            ) as c:
                for url in urls:
                    try:
                        r = await c.get(url)
                        if r.status_code == 200:
                            ip = _parse_ip(r.text)
                            if ip:
                                return ip
                    except Exception:
                        continue
        except Exception:
            pass
        return None

    try:
        return await asyncio.wait_for(_run(), timeout=budget)
    except asyncio.TimeoutError:
        log.debug("get_direct_ip timed out after %.1fs", budget)
        return None