Add NordVPN proxy rotation for geo-distributed web fetching

3 proxies: Tokyo (.154:3128), London (.71:3128), Sydney (.189:3128)
Random rotation per crawl — bypasses geo-blocks and rate limits
This commit is contained in:
drjones
2026-08-04 06:20:37 -07:00
parent da75b635f4
commit 966b0e8d59

View File

@@ -159,11 +159,24 @@ def semantic_search(q: str = Query(...), limit: int = 10):
import re import re
import html as html_mod import html as html_mod
import random
# NordVPN HTTP proxies for geo-distributed fetching
PROXIES = [
"http://10.30.20.154:3128", # Tokyo, Japan
"http://10.30.20.71:3128", # London, UK
"http://10.30.20.189:3128", # Sydney, Australia
]
def _fetch_and_index(url: str, category: str = ""): def _fetch_and_index(url: str, category: str = ""):
"""Fetch a URL, extract text, and index into OpenSearch immediately.""" """Fetch a URL, extract text, and index into OpenSearch immediately."""
proxy_url = random.choice(PROXIES)
try: try:
r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0) # Create a per-request client with proxy
proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0)
r = proxy_client.get(url,
headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"},
)
if r.status_code != 200: if r.status_code != 200:
return None return None
html_text = r.text html_text = r.text