Early-exit validation: stop once enough valid proxies found
- validate_proxies: target= param — once N valid found, remaining tasks skip the HTTP call and drain immediately (no wasted time) - _build_pool: target = max(min_pool_size, chain_length * 3) - Settings: min_pool_size (default 20), max_candidates default 400->200 - UI: Stop after N valid / Max candidates fields in Settings tab - README: updated settings table - 1 new test for early-exit with unreachable target Made-with: Cursor
This commit is contained in:
@@ -44,7 +44,13 @@ async def validate_proxies(
|
||||
concurrency: int,
|
||||
timeout_seconds: float,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
target: int = 0,
|
||||
) -> list[str]:
|
||||
"""Validate proxies concurrently.
|
||||
|
||||
Args:
|
||||
target: Stop as soon as this many valid proxies are found (0 = test all).
|
||||
"""
|
||||
if not proxy_urls:
|
||||
return []
|
||||
sem = asyncio.Semaphore(max(1, concurrency))
|
||||
@@ -57,30 +63,46 @@ async def validate_proxies(
|
||||
cap = min(600.0, float(waves) * float(timeout_seconds) + 45.0)
|
||||
cap = max(90.0, cap)
|
||||
log.info(
|
||||
"validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs check=%s",
|
||||
"validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs target=%s check=%s",
|
||||
total,
|
||||
c,
|
||||
timeout_seconds,
|
||||
cap,
|
||||
target if target else "all",
|
||||
(check_url[:70] + "…") if len(check_url) > 70 else check_url,
|
||||
)
|
||||
|
||||
last_prog_t = 0.0
|
||||
prog_step = max(1, total // 120)
|
||||
enough = False # set True once target reached; tasks skip HTTP call and drain fast
|
||||
|
||||
async def one(u: str) -> None:
|
||||
nonlocal done, last_prog_t
|
||||
nonlocal done, last_prog_t, enough
|
||||
# Skip cost of the HTTP check once we already have enough
|
||||
if enough:
|
||||
async with lock:
|
||||
done += 1
|
||||
if on_progress and done == total:
|
||||
on_progress(done, total)
|
||||
return
|
||||
async with sem:
|
||||
good = await _check_one(u, check_url, timeout_seconds)
|
||||
if enough: # re-check after potentially waiting on semaphore
|
||||
good = False
|
||||
else:
|
||||
good = await _check_one(u, check_url, timeout_seconds)
|
||||
async with lock:
|
||||
done += 1
|
||||
if good:
|
||||
ok.append(u)
|
||||
if target and len(ok) >= target:
|
||||
enough = True
|
||||
log.info("validate_proxies: target %d reached after %d tested", target, done)
|
||||
if on_progress:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
done == total
|
||||
or done == 1
|
||||
or enough
|
||||
or done % prog_step == 0
|
||||
or (now - last_prog_t) >= 0.1
|
||||
):
|
||||
@@ -98,7 +120,7 @@ async def validate_proxies(
|
||||
cap,
|
||||
total,
|
||||
)
|
||||
log.info("validate_proxies: finished ok=%d / %d", len(ok), total)
|
||||
log.info("validate_proxies: finished ok=%d / %d tested", len(ok), done)
|
||||
return ok
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user