Image scraping + Postgres env vars + search results include images

- _fetch_and_index now extracts img src URLs, filters junk, stores in OpenSearch
- get_db() uses PGHOST/PGUSER/etc env vars for Docker compatibility
- search endpoint returns images + image_count in hits
- Docker run command passes all PG env vars
- Discovery script runs via SSH to CT to avoid VPN issues

backend.py

@@ -20,8 +20,8 @@ app = FastAPI(title="AI Research Engine")
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
 
 # ── Config ──────────────────────────────────────────────────
-YACY_URL       = os.getenv("YACY_URL", "http://host.docker.internal:8090")
+YACY_URL       = os.getenv("YACY_URL", "http://localhost:8090")
-OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200")
+OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200")
 QDRANT_URL     = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
 OLLAMA_URL     = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434")
 OLLAMA_MODEL   = os.getenv("OLLAMA_MODEL", "ornith:latest")
@@ -53,9 +53,9 @@ DEFAULT_TIMEOUT = 300  # 5 minutes per API call max
 # DB
 def get_db():
     return psycopg2.connect(
-        host="host.docker.internal", port=5432,
+        host=os.getenv("PGHOST", "localhost"), port=int(os.getenv("PGPORT","5432")),
-        user="research", password="ResearchDB2026!",
+        user=os.getenv("PGUSER", "research"), password=os.getenv("PGPASSWORD", "ResearchDB2026!"),
-        database="research_engine"
+        database=os.getenv("PGDATABASE", "research_engine")
     )
 
 client = httpx.Client(timeout=30.0)
@@ -107,13 +107,26 @@ def _get_embedding(text: str) -> list:
     except: return []
 
 def _fetch_and_index(url: str, category: str = ""):
-    """Fetch a URL through round-robin proxy, extract text, index into OpenSearch + Qdrant."""
+    """Fetch URL through proxy, extract text + images, index into OpenSearch + Qdrant."""
     proxy_name, proxy_url = _next_proxy()
     try:
         pc = httpx.Client(proxy=proxy_url, timeout=60.0)
         r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"})
         if r.status_code != 200: return None
         html_text = r.text
+        # Extract image URLs
+        image_urls = []
+        for m in re.finditer(r'<img[^>]+src=["\x27]([^"\x27]+)["\x27]', html_text, re.IGNORECASE):
+            src = m.group(1)
+            if src.startswith("//"): src = "https:" + src
+            elif src.startswith("/"):
+                base = urlparse(url)
+                src = f"{base.scheme}://{base.netloc}{src}"
+            elif not src.startswith("http"): continue
+            if any(x in src.lower() for x in ["icon","logo","pixel","tracking","analytics","1x1","spacer","blank","avatar","badge"]): continue
+            image_urls.append(src)
+        image_urls = list(dict.fromkeys(image_urls))[:10]
+        # Text extraction
         text = re.sub(r'<script[^>]*>.*?</script>','',html_text,flags=re.DOTALL|re.IGNORECASE)
         text = re.sub(r'<style[^>]*>.*?</style>','',text,flags=re.DOTALL|re.IGNORECASE)
         text = re.sub(r'<[^>]+>',' ',text)
@@ -121,14 +134,14 @@ def _fetch_and_index(url: str, category: str = ""):
         text = html_mod.unescape(text)
         title_m = re.search(r'<title[^>]*>(.*?)</title>',html_text,re.IGNORECASE|re.DOTALL)
         title = html_mod.unescape(title_m.group(1).strip()) if title_m else url
-        desc_m = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE)
+        desc_m = re.search(r'<meta[^>]+name=["\x27]description["\x27][^>]+content=["\x27]([^"\x27]+)',html_text,re.IGNORECASE)
         excerpt = desc_m.group(1)[:500] if desc_m else text[:500]
         domain = urlparse(url).netloc
         if not category:
             for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items():
                 if k in domain: category = v; break
             if not category: category = "web"
-        doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
+        doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"images":image_urls,"image_count":len(image_urls),"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
         _ensure_index()
         client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"})
         emb = _get_embedding(excerpt[:1000])
@@ -136,7 +149,7 @@ def _fetch_and_index(url: str, category: str = ""):
             try:
                 client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]})
             except: pass
-        return {"title":title,"domain":domain,"category":category,"size":len(text)}
+        return {"title":title,"domain":domain,"category":category,"size":len(text),"images":len(image_urls),"proxy":proxy_name}
     except: return None
 
 
@@ -383,6 +396,8 @@ def status():
 
 @app.get("/api/search")
 def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10):
+    if not q or not q.strip():
+        return {"query": "", "total": 0, "hits": [], "hint": "Empty query — provide a search term"}
     user = _auth(request)
     _track_usage(user["id"], "search_web", request)
     _ensure_index()
@@ -394,7 +409,7 @@ def search_web(request: Request, q: str = Query(...), category: str = "", limit:
         hits = []
         for h in result.get("hits",{}).get("hits",[]):
             src = h["_source"]
-            hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]})
+            hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"],"images":src.get("images",[]),"image_count":src.get("image_count",0)})
         return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits}
     except Exception as e:
         return {"query":q,"total":0,"hits":[],"note":str(e)}

docker-compose.yml

@@ -56,7 +56,46 @@ services:
       timeout: 5s
       retries: 3
 
+  # ── Database ───────────────────────────────────────────────
+  postgres:
+    image: postgres:16-alpine
+    container_name: postgres
+    environment:
+      - POSTGRES_USER=research
+      - POSTGRES_PASSWORD=ResearchDB2026!
+      - POSTGRES_DB=research_engine
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "research"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ── API + Dashboard ────────────────────────────────────────
+  backend:
+    build: .
+    container_name: research-backend
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      opensearch:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
 volumes:
   yacy_data:
   opensearch_data:
   redis_data:
+  postgres_data: