diff --git a/backend.py b/backend.py index 4648d26..10b51a7 100644 --- a/backend.py +++ b/backend.py @@ -159,11 +159,24 @@ def semantic_search(q: str = Query(...), limit: int = 10): import re import html as html_mod +import random + +# NordVPN HTTP proxies for geo-distributed fetching +PROXIES = [ + "http://10.30.20.154:3128", # Tokyo, Japan + "http://10.30.20.71:3128", # London, UK + "http://10.30.20.189:3128", # Sydney, Australia +] def _fetch_and_index(url: str, category: str = ""): """Fetch a URL, extract text, and index into OpenSearch immediately.""" + proxy_url = random.choice(PROXIES) try: - r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0) + # Create a per-request client with proxy + proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0) + r = proxy_client.get(url, + headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, + ) if r.status_code != 200: return None html_text = r.text