107 lines
3.1 KiB
Python
107 lines
3.1 KiB
Python
"""Fetch proxy lists for UI picker and shared pool building."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Callable
|
|
|
|
from .config import Settings
|
|
from .fetcher import fetch_proxy_json, normalize_entries
|
|
from .validator import validate_proxies
|
|
|
|
log = logging.getLogger(__name__)
|
|
_POOL_EXEC = ThreadPoolExecutor(max_workers=6, thread_name_prefix="pool_ops")
|
|
|
|
PICKER_DISPLAY_CAP = 200
|
|
|
|
|
|
class ProxySourceCache:
|
|
"""Cache full source lists; each UI pull shows a new random batch."""
|
|
|
|
def __init__(self) -> None:
|
|
self._full: list[str] = []
|
|
self.total_cached: int = 0
|
|
|
|
def clear(self) -> None:
|
|
self._full.clear()
|
|
self.total_cached = 0
|
|
|
|
@property
|
|
def loaded(self) -> bool:
|
|
return bool(self._full)
|
|
|
|
def load(self, sources: list[str], prefer_elite: bool,
|
|
on_status: Callable[[str], None] | None = None) -> int:
|
|
self._full = fetch_raw_proxies(sources, prefer_elite, on_status=on_status)
|
|
self.total_cached = len(self._full)
|
|
return self.total_cached
|
|
|
|
def random_batch(self, cap: int = PICKER_DISPLAY_CAP) -> list[str]:
|
|
if not self._full:
|
|
return []
|
|
k = min(cap, len(self._full))
|
|
return random.sample(self._full, k)
|
|
|
|
|
|
def fetch_raw_proxies(
|
|
sources: list[str],
|
|
prefer_elite: bool,
|
|
on_status: Callable[[str], None] | None = None,
|
|
) -> list[str]:
|
|
"""Blocking fetch from all JSON sources; deduped, not validated."""
|
|
if not sources:
|
|
sources = list(Settings().sources)
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for url in sources:
|
|
if on_status:
|
|
on_status(f"Fetching {url[:70]}…")
|
|
try:
|
|
rows = fetch_proxy_json(url, timeout=45.0)
|
|
entries = normalize_entries(rows, prefer_elite)
|
|
if on_status:
|
|
on_status(f" → {len(entries)} proxies")
|
|
for u in entries:
|
|
if u not in seen:
|
|
seen.add(u)
|
|
out.append(u)
|
|
except Exception as e:
|
|
log.warning("fetch_raw_proxies %s: %s", url[:60], e)
|
|
if on_status:
|
|
on_status(f" → error: {e!s}")
|
|
return out
|
|
|
|
|
|
async def validate_proxy_subset(
|
|
urls: list[str],
|
|
settings: Settings,
|
|
target: int = 0,
|
|
on_progress: Callable[[int, int], None] | None = None,
|
|
) -> list[str]:
|
|
if not urls:
|
|
return []
|
|
sample = list(urls)
|
|
random.shuffle(sample)
|
|
if len(sample) > settings.max_candidates:
|
|
sample = sample[: settings.max_candidates]
|
|
tgt = target or settings.min_pool_size
|
|
return await validate_proxies(
|
|
sample,
|
|
settings.ip_check_url,
|
|
settings.validation_concurrency,
|
|
settings.validation_timeout_seconds,
|
|
on_progress=on_progress,
|
|
target=tgt,
|
|
)
|
|
|
|
|
|
def run_validate_sync(
|
|
urls: list[str],
|
|
settings: Settings,
|
|
target: int = 0,
|
|
on_progress: Callable[[int, int], None] | None = None,
|
|
) -> list[str]:
|
|
return asyncio.run(validate_proxy_subset(urls, settings, target=target, on_progress=on_progress))
|