From 966b0e8d59046c98fdfe0600eef5e5e313768ceb Mon Sep 17 00:00:00 2001 From: drjones Date: Tue, 4 Aug 2026 06:20:37 -0700 Subject: [PATCH] Add NordVPN proxy rotation for geo-distributed web fetching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3 proxies: Tokyo (.154:3128), London (.71:3128), Sydney (.189:3128) Random rotation per crawl — bypasses geo-blocks and rate limits --- backend.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend.py b/backend.py index 4648d26..10b51a7 100644 --- a/backend.py +++ b/backend.py @@ -159,11 +159,24 @@ def semantic_search(q: str = Query(...), limit: int = 10): import re import html as html_mod +import random + +# NordVPN HTTP proxies for geo-distributed fetching +PROXIES = [ + "http://10.30.20.154:3128", # Tokyo, Japan + "http://10.30.20.71:3128", # London, UK + "http://10.30.20.189:3128", # Sydney, Australia +] def _fetch_and_index(url: str, category: str = ""): """Fetch a URL, extract text, and index into OpenSearch immediately.""" + proxy_url = random.choice(PROXIES) try: - r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0) + # Create a per-request client with proxy + proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0) + r = proxy_client.get(url, + headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, + ) if r.status_code != 200: return None html_text = r.text