"""Fetch proxy lists for UI picker and shared pool building.""" from __future__ import annotations import asyncio import logging import random from concurrent.futures import ThreadPoolExecutor from typing import Callable from .config import Settings from .fetcher import fetch_proxy_json, normalize_entries from .validator import validate_proxies log = logging.getLogger(__name__) _POOL_EXEC = ThreadPoolExecutor(max_workers=6, thread_name_prefix="pool_ops") PICKER_DISPLAY_CAP = 200 class ProxySourceCache: """Cache full source lists; each UI pull shows a new random batch.""" def __init__(self) -> None: self._full: list[str] = [] self.total_cached: int = 0 def clear(self) -> None: self._full.clear() self.total_cached = 0 @property def loaded(self) -> bool: return bool(self._full) def load(self, sources: list[str], prefer_elite: bool, on_status: Callable[[str], None] | None = None) -> int: self._full = fetch_raw_proxies(sources, prefer_elite, on_status=on_status) self.total_cached = len(self._full) return self.total_cached def random_batch(self, cap: int = PICKER_DISPLAY_CAP) -> list[str]: if not self._full: return [] k = min(cap, len(self._full)) return random.sample(self._full, k) def fetch_raw_proxies( sources: list[str], prefer_elite: bool, on_status: Callable[[str], None] | None = None, ) -> list[str]: """Blocking fetch from all JSON sources; deduped, not validated.""" if not sources: sources = list(Settings().sources) seen: set[str] = set() out: list[str] = [] for url in sources: if on_status: on_status(f"Fetching {url[:70]}…") try: rows = fetch_proxy_json(url, timeout=45.0) entries = normalize_entries(rows, prefer_elite) if on_status: on_status(f" → {len(entries)} proxies") for u in entries: if u not in seen: seen.add(u) out.append(u) except Exception as e: log.warning("fetch_raw_proxies %s: %s", url[:60], e) if on_status: on_status(f" → error: {e!s}") return out async def validate_proxy_subset( urls: list[str], settings: Settings, target: int = 0, on_progress: Callable[[int, int], None] | None = None, ) -> list[str]: if not urls: return [] sample = list(urls) random.shuffle(sample) if len(sample) > settings.max_candidates: sample = sample[: settings.max_candidates] tgt = target or settings.min_pool_size return await validate_proxies( sample, settings.ip_check_url, settings.validation_concurrency, settings.validation_timeout_seconds, on_progress=on_progress, target=tgt, ) def run_validate_sync( urls: list[str], settings: Settings, target: int = 0, on_progress: Callable[[int, int], None] | None = None, ) -> list[str]: return asyncio.run(validate_proxy_subset(urls, settings, target=target, on_progress=on_progress))