Add NordVPN proxy rotation for geo-distributed web fetching
3 proxies: Tokyo (.154:3128), London (.71:3128), Sydney (.189:3128) Random rotation per crawl — bypasses geo-blocks and rate limits
This commit is contained in:
15
backend.py
15
backend.py
@@ -159,11 +159,24 @@ def semantic_search(q: str = Query(...), limit: int = 10):
|
||||
|
||||
import re
|
||||
import html as html_mod
|
||||
import random
|
||||
|
||||
# NordVPN HTTP proxies for geo-distributed fetching
|
||||
PROXIES = [
|
||||
"http://10.30.20.154:3128", # Tokyo, Japan
|
||||
"http://10.30.20.71:3128", # London, UK
|
||||
"http://10.30.20.189:3128", # Sydney, Australia
|
||||
]
|
||||
|
||||
def _fetch_and_index(url: str, category: str = ""):
|
||||
"""Fetch a URL, extract text, and index into OpenSearch immediately."""
|
||||
proxy_url = random.choice(PROXIES)
|
||||
try:
|
||||
r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0)
|
||||
# Create a per-request client with proxy
|
||||
proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0)
|
||||
r = proxy_client.get(url,
|
||||
headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
html_text = r.text
|
||||
|
||||
Reference in New Issue
Block a user