Image scraping + Postgres env vars + search results include images
- _fetch_and_index now extracts img src URLs, filters junk, stores in OpenSearch - get_db() uses PGHOST/PGUSER/etc env vars for Docker compatibility - search endpoint returns images + image_count in hits - Docker run command passes all PG env vars - Discovery script runs via SSH to CT to avoid VPN issues
This commit is contained in:
drjones
35
backend.py
35
backend.py
@@ -20,8 +20,8 @@ app = FastAPI(title="AI Research Engine")
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
||||
|
||||
# ── Config ──────────────────────────────────────────────────
|
||||
YACY_URL = os.getenv("YACY_URL", "http://host.docker.internal:8090")
|
||||
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200")
|
||||
YACY_URL = os.getenv("YACY_URL", "http://localhost:8090")
|
||||
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200")
|
||||
QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434")
|
||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "ornith:latest")
|
||||
@@ -53,9 +53,9 @@ DEFAULT_TIMEOUT = 300 # 5 minutes per API call max
|
||||
# DB
|
||||
def get_db():
|
||||
return psycopg2.connect(
|
||||
host="host.docker.internal", port=5432,
|
||||
user="research", password="ResearchDB2026!",
|
||||
database="research_engine"
|
||||
host=os.getenv("PGHOST", "localhost"), port=int(os.getenv("PGPORT","5432")),
|
||||
user=os.getenv("PGUSER", "research"), password=os.getenv("PGPASSWORD", "ResearchDB2026!"),
|
||||
database=os.getenv("PGDATABASE", "research_engine")
|
||||
)
|
||||
|
||||
client = httpx.Client(timeout=30.0)
|
||||
@@ -107,13 +107,26 @@ def _get_embedding(text: str) -> list:
|
||||
except: return []
|
||||
|
||||
def _fetch_and_index(url: str, category: str = ""):
|
||||
"""Fetch a URL through round-robin proxy, extract text, index into OpenSearch + Qdrant."""
|
||||
"""Fetch URL through proxy, extract text + images, index into OpenSearch + Qdrant."""
|
||||
proxy_name, proxy_url = _next_proxy()
|
||||
try:
|
||||
pc = httpx.Client(proxy=proxy_url, timeout=60.0)
|
||||
r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"})
|
||||
if r.status_code != 200: return None
|
||||
html_text = r.text
|
||||
# Extract image URLs
|
||||
image_urls = []
|
||||
for m in re.finditer(r'<img[^>]+src=["\x27]([^"\x27]+)["\x27]', html_text, re.IGNORECASE):
|
||||
src = m.group(1)
|
||||
if src.startswith("//"): src = "https:" + src
|
||||
elif src.startswith("/"):
|
||||
base = urlparse(url)
|
||||
src = f"{base.scheme}://{base.netloc}{src}"
|
||||
elif not src.startswith("http"): continue
|
||||
if any(x in src.lower() for x in ["icon","logo","pixel","tracking","analytics","1x1","spacer","blank","avatar","badge"]): continue
|
||||
image_urls.append(src)
|
||||
image_urls = list(dict.fromkeys(image_urls))[:10]
|
||||
# Text extraction
|
||||
text = re.sub(r'<script[^>]*>.*?</script>','',html_text,flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<style[^>]*>.*?</style>','',text,flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<[^>]+>',' ',text)
|
||||
@@ -121,14 +134,14 @@ def _fetch_and_index(url: str, category: str = ""):
|
||||
text = html_mod.unescape(text)
|
||||
title_m = re.search(r'<title[^>]*>(.*?)</title>',html_text,re.IGNORECASE|re.DOTALL)
|
||||
title = html_mod.unescape(title_m.group(1).strip()) if title_m else url
|
||||
desc_m = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE)
|
||||
desc_m = re.search(r'<meta[^>]+name=["\x27]description["\x27][^>]+content=["\x27]([^"\x27]+)',html_text,re.IGNORECASE)
|
||||
excerpt = desc_m.group(1)[:500] if desc_m else text[:500]
|
||||
domain = urlparse(url).netloc
|
||||
if not category:
|
||||
for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items():
|
||||
if k in domain: category = v; break
|
||||
if not category: category = "web"
|
||||
doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
|
||||
doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"images":image_urls,"image_count":len(image_urls),"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
|
||||
_ensure_index()
|
||||
client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"})
|
||||
emb = _get_embedding(excerpt[:1000])
|
||||
@@ -136,7 +149,7 @@ def _fetch_and_index(url: str, category: str = ""):
|
||||
try:
|
||||
client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]})
|
||||
except: pass
|
||||
return {"title":title,"domain":domain,"category":category,"size":len(text)}
|
||||
return {"title":title,"domain":domain,"category":category,"size":len(text),"images":len(image_urls),"proxy":proxy_name}
|
||||
except: return None
|
||||
|
||||
|
||||
@@ -383,6 +396,8 @@ def status():
|
||||
|
||||
@app.get("/api/search")
|
||||
def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10):
|
||||
if not q or not q.strip():
|
||||
return {"query": "", "total": 0, "hits": [], "hint": "Empty query — provide a search term"}
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "search_web", request)
|
||||
_ensure_index()
|
||||
@@ -394,7 +409,7 @@ def search_web(request: Request, q: str = Query(...), category: str = "", limit:
|
||||
hits = []
|
||||
for h in result.get("hits",{}).get("hits",[]):
|
||||
src = h["_source"]
|
||||
hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]})
|
||||
hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"],"images":src.get("images",[]),"image_count":src.get("image_count",0)})
|
||||
return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits}
|
||||
except Exception as e:
|
||||
return {"query":q,"total":0,"hits":[],"note":str(e)}
|
||||
|
||||
@@ -56,7 +56,46 @@ services:
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ── Database ───────────────────────────────────────────────
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: postgres
|
||||
environment:
|
||||
- POSTGRES_USER=research
|
||||
- POSTGRES_PASSWORD=ResearchDB2026!
|
||||
- POSTGRES_DB=research_engine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "research"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── API + Dashboard ────────────────────────────────────────
|
||||
backend:
|
||||
build: .
|
||||
container_name: research-backend
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
yacy_data:
|
||||
opensearch_data:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
Reference in New Issue
Block a user