Add NordVPN proxy rotation for geo-distributed web fetching
3 proxies: Tokyo (.154:3128), London (.71:3128), Sydney (.189:3128) Random rotation per crawl — bypasses geo-blocks and rate limits
This commit is contained in:
15
backend.py
15
backend.py
@@ -159,11 +159,24 @@ def semantic_search(q: str = Query(...), limit: int = 10):
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
import html as html_mod
|
import html as html_mod
|
||||||
|
import random
|
||||||
|
|
||||||
|
# NordVPN HTTP proxies for geo-distributed fetching
|
||||||
|
PROXIES = [
|
||||||
|
"http://10.30.20.154:3128", # Tokyo, Japan
|
||||||
|
"http://10.30.20.71:3128", # London, UK
|
||||||
|
"http://10.30.20.189:3128", # Sydney, Australia
|
||||||
|
]
|
||||||
|
|
||||||
def _fetch_and_index(url: str, category: str = ""):
|
def _fetch_and_index(url: str, category: str = ""):
|
||||||
"""Fetch a URL, extract text, and index into OpenSearch immediately."""
|
"""Fetch a URL, extract text, and index into OpenSearch immediately."""
|
||||||
|
proxy_url = random.choice(PROXIES)
|
||||||
try:
|
try:
|
||||||
r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0)
|
# Create a per-request client with proxy
|
||||||
|
proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0)
|
||||||
|
r = proxy_client.get(url,
|
||||||
|
headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"},
|
||||||
|
)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
return None
|
return None
|
||||||
html_text = r.text
|
html_text = r.text
|
||||||
|
|||||||
Reference in New Issue
Block a user