first commit

proxy_chain_manager/pool_ops.py

@@ -0,0 +1,106 @@
+"""Fetch proxy lists for UI picker and shared pool building."""
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+
+from .config import Settings
+from .fetcher import fetch_proxy_json, normalize_entries
+from .validator import validate_proxies
+
+log = logging.getLogger(__name__)
+_POOL_EXEC = ThreadPoolExecutor(max_workers=6, thread_name_prefix="pool_ops")
+
+PICKER_DISPLAY_CAP = 200
+
+
+class ProxySourceCache:
+    """Cache full source lists; each UI pull shows a new random batch."""
+
+    def __init__(self) -> None:
+        self._full: list[str] = []
+        self.total_cached: int = 0
+
+    def clear(self) -> None:
+        self._full.clear()
+        self.total_cached = 0
+
+    @property
+    def loaded(self) -> bool:
+        return bool(self._full)
+
+    def load(self, sources: list[str], prefer_elite: bool,
+             on_status: Callable[[str], None] | None = None) -> int:
+        self._full = fetch_raw_proxies(sources, prefer_elite, on_status=on_status)
+        self.total_cached = len(self._full)
+        return self.total_cached
+
+    def random_batch(self, cap: int = PICKER_DISPLAY_CAP) -> list[str]:
+        if not self._full:
+            return []
+        k = min(cap, len(self._full))
+        return random.sample(self._full, k)
+
+
+def fetch_raw_proxies(
+    sources: list[str],
+    prefer_elite: bool,
+    on_status: Callable[[str], None] | None = None,
+) -> list[str]:
+    """Blocking fetch from all JSON sources; deduped, not validated."""
+    if not sources:
+        sources = list(Settings().sources)
+    seen: set[str] = set()
+    out: list[str] = []
+    for url in sources:
+        if on_status:
+            on_status(f"Fetching {url[:70]}…")
+        try:
+            rows = fetch_proxy_json(url, timeout=45.0)
+            entries = normalize_entries(rows, prefer_elite)
+            if on_status:
+                on_status(f"  → {len(entries)} proxies")
+            for u in entries:
+                if u not in seen:
+                    seen.add(u)
+                    out.append(u)
+        except Exception as e:
+            log.warning("fetch_raw_proxies %s: %s", url[:60], e)
+            if on_status:
+                on_status(f"  → error: {e!s}")
+    return out
+
+
+async def validate_proxy_subset(
+    urls: list[str],
+    settings: Settings,
+    target: int = 0,
+    on_progress: Callable[[int, int], None] | None = None,
+) -> list[str]:
+    if not urls:
+        return []
+    sample = list(urls)
+    random.shuffle(sample)
+    if len(sample) > settings.max_candidates:
+        sample = sample[: settings.max_candidates]
+    tgt = target or settings.min_pool_size
+    return await validate_proxies(
+        sample,
+        settings.ip_check_url,
+        settings.validation_concurrency,
+        settings.validation_timeout_seconds,
+        on_progress=on_progress,
+        target=tgt,
+    )
+
+
+def run_validate_sync(
+    urls: list[str],
+    settings: Settings,
+    target: int = 0,
+    on_progress: Callable[[int, int], None] | None = None,
+) -> list[str]:
+    return asyncio.run(validate_proxy_subset(urls, settings, target=target, on_progress=on_progress))