From 9439037cacd9dcff791125f83534dd9dbf9b82bd Mon Sep 17 00:00:00 2001 From: Dr Jones Date: Thu, 16 Apr 2026 02:31:40 -0700 Subject: [PATCH] Early-exit validation: stop once enough valid proxies found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - validate_proxies: target= param — once N valid found, remaining tasks skip the HTTP call and drain immediately (no wasted time) - _build_pool: target = max(min_pool_size, chain_length * 3) - Settings: min_pool_size (default 20), max_candidates default 400->200 - UI: Stop after N valid / Max candidates fields in Settings tab - README: updated settings table - 1 new test for early-exit with unreachable target Made-with: Cursor --- README.md | 3 ++- proxy_chain_manager/app.py | 7 +++++-- proxy_chain_manager/config.py | 19 +++++++++++++++++-- proxy_chain_manager/service.py | 2 ++ proxy_chain_manager/validator.py | 30 ++++++++++++++++++++++++++---- tests/test_proxy_god.py | 14 ++++++++++++++ 6 files changed, 66 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b31bb2f..b313373 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,8 @@ Hits config sanitization, fetcher smoke, validator timeouts, and a live `get_dir | Health check | `180` s | How often the exit **gets re-proven** | | Pool refresh | `1800` s | How often lists get **re-fetched** and re-tested | | Concurrency | `64` | How hard you **spam** validation | -| Max candidates | `400` | Cap per cycle before testing | +| Stop after N valid | `20` | **Early-exit** — stop testing as soon as N good proxies found | +| Max candidates | `200` | Cap per cycle (shuffled, so random sample) | | Timeout | `12` s | Per-proxy **patience** | | Kill-switch | ON | **All-or-nothing** outbound (when Admin + firewall engaged) | diff --git a/proxy_chain_manager/app.py b/proxy_chain_manager/app.py index deb4b6a..072f341 100644 --- a/proxy_chain_manager/app.py +++ b/proxy_chain_manager/app.py @@ -759,8 +759,10 @@ def main() -> None: val_sec = _section("Validation") _fld(val_sec, "Concurrent validators", "conc", str(s.validation_concurrency)) - _fld(val_sec, "Max candidates per cycle", "maxc", str(s.max_candidates), - "(random sample from fetch)") + _fld(val_sec, "Stop after N valid proxies found", "minpool", str(s.min_pool_size), + "(early-exit — stop testing once enough found)") + _fld(val_sec, "Max candidates to test per cycle", "maxc", str(s.max_candidates), + "(shuffled random sample from fetch)") _fld(val_sec, "Per-proxy timeout (sec)", "timeout", str(s.validation_timeout_seconds)) sec_sec = _section("Security") @@ -808,6 +810,7 @@ def main() -> None: health_check_seconds=min(3600, max(10, int(entries["health"].get().strip()))), full_refresh_seconds=min(86400, max(60, int(entries["refresh"].get().strip()))), validation_concurrency=max(1, int(entries["conc"].get().strip())), + min_pool_size=max(3, int(entries["minpool"].get().strip())), max_candidates=max(10, int(entries["maxc"].get().strip())), validation_timeout_seconds=min(120.0, max(2.0, float(entries["timeout"].get().strip()))), prefer_elite=bool(elite_var.get()), diff --git a/proxy_chain_manager/config.py b/proxy_chain_manager/config.py index 1d2aee1..d3d6b47 100644 --- a/proxy_chain_manager/config.py +++ b/proxy_chain_manager/config.py @@ -126,7 +126,8 @@ class Settings: # ── validation ──────────────────────────────────────────────────────── validation_concurrency: int = 64 - max_candidates: int = 400 + max_candidates: int = 200 # cap before validation (shuffled, so random sample) + min_pool_size: int = 20 # stop validating once this many good proxies found validation_timeout_seconds: float = 12.0 prefer_elite: bool = False @@ -204,7 +205,21 @@ def sanitize_settings(s: Settings) -> tuple[Settings, bool]: s.max_candidates = 10 changed = True except (TypeError, ValueError): - s.max_candidates = 400 + s.max_candidates = 200 + changed = True + try: + mps = int(s.min_pool_size) + if mps < 3: + s.min_pool_size = 3 + changed = True + elif mps > 500: + s.min_pool_size = 500 + changed = True + except (TypeError, ValueError): + s.min_pool_size = 20 + changed = True + if not hasattr(s, "min_pool_size"): + s.min_pool_size = 20 changed = True try: vt = float(s.validation_timeout_seconds) diff --git a/proxy_chain_manager/service.py b/proxy_chain_manager/service.py index 0d3eb9d..f4036a4 100644 --- a/proxy_chain_manager/service.py +++ b/proxy_chain_manager/service.py @@ -579,12 +579,14 @@ class ChainService: def on_prog(done: int, total: int) -> None: self._notify({"type": "validate_progress", "done": done, "total": total}) + target = max(s.min_pool_size, s.chain_length * 3) good = await validate_proxies( unique, s.ip_check_url, s.validation_concurrency, s.validation_timeout_seconds, on_progress=on_prog, + target=target, ) random.shuffle(good) diff --git a/proxy_chain_manager/validator.py b/proxy_chain_manager/validator.py index ab08da5..8be332e 100644 --- a/proxy_chain_manager/validator.py +++ b/proxy_chain_manager/validator.py @@ -44,7 +44,13 @@ async def validate_proxies( concurrency: int, timeout_seconds: float, on_progress: Callable[[int, int], None] | None = None, + target: int = 0, ) -> list[str]: + """Validate proxies concurrently. + + Args: + target: Stop as soon as this many valid proxies are found (0 = test all). + """ if not proxy_urls: return [] sem = asyncio.Semaphore(max(1, concurrency)) @@ -57,30 +63,46 @@ async def validate_proxies( cap = min(600.0, float(waves) * float(timeout_seconds) + 45.0) cap = max(90.0, cap) log.info( - "validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs check=%s", + "validate_proxies: n=%d concurrency=%d per_timeout=%.1fs wall_cap~=%.0fs target=%s check=%s", total, c, timeout_seconds, cap, + target if target else "all", (check_url[:70] + "…") if len(check_url) > 70 else check_url, ) last_prog_t = 0.0 prog_step = max(1, total // 120) + enough = False # set True once target reached; tasks skip HTTP call and drain fast async def one(u: str) -> None: - nonlocal done, last_prog_t + nonlocal done, last_prog_t, enough + # Skip cost of the HTTP check once we already have enough + if enough: + async with lock: + done += 1 + if on_progress and done == total: + on_progress(done, total) + return async with sem: - good = await _check_one(u, check_url, timeout_seconds) + if enough: # re-check after potentially waiting on semaphore + good = False + else: + good = await _check_one(u, check_url, timeout_seconds) async with lock: done += 1 if good: ok.append(u) + if target and len(ok) >= target: + enough = True + log.info("validate_proxies: target %d reached after %d tested", target, done) if on_progress: now = time.monotonic() if ( done == total or done == 1 + or enough or done % prog_step == 0 or (now - last_prog_t) >= 0.1 ): @@ -98,7 +120,7 @@ async def validate_proxies( cap, total, ) - log.info("validate_proxies: finished ok=%d / %d", len(ok), total) + log.info("validate_proxies: finished ok=%d / %d tested", len(ok), done) return ok diff --git a/tests/test_proxy_god.py b/tests/test_proxy_god.py index 0ad3945..1b1850f 100644 --- a/tests/test_proxy_god.py +++ b/tests/test_proxy_god.py @@ -129,6 +129,20 @@ class TestValidator(unittest.TestCase): self.assertEqual(asyncio.run(run()), []) + def test_validate_early_exit_target(self) -> None: + """Once target is reached remaining tasks should be skipped (all dead here → no early exit but no crash).""" + async def run() -> list[str]: + return await validate_proxies( + ["http://127.0.0.1:1"] * 10, + "https://api.ipify.org?format=json", + 4, + 2.0, + target=2, # target unreachable — should still return [] cleanly + ) + + result = asyncio.run(run()) + self.assertEqual(result, []) + def test_get_direct_ip_network(self) -> None: async def run() -> str | None: return await get_direct_ip("https://api.ipify.org?format=json", 12.0)