diff --git a/backend.py b/backend.py index ac98223..f996018 100644 --- a/backend.py +++ b/backend.py @@ -20,8 +20,8 @@ app = FastAPI(title="AI Research Engine") app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) # ── Config ────────────────────────────────────────────────── -YACY_URL = os.getenv("YACY_URL", "http://host.docker.internal:8090") -OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200") +YACY_URL = os.getenv("YACY_URL", "http://localhost:8090") +OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200") QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333") OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "ornith:latest") @@ -53,9 +53,9 @@ DEFAULT_TIMEOUT = 300 # 5 minutes per API call max # DB def get_db(): return psycopg2.connect( - host="host.docker.internal", port=5432, - user="research", password="ResearchDB2026!", - database="research_engine" + host=os.getenv("PGHOST", "localhost"), port=int(os.getenv("PGPORT","5432")), + user=os.getenv("PGUSER", "research"), password=os.getenv("PGPASSWORD", "ResearchDB2026!"), + database=os.getenv("PGDATABASE", "research_engine") ) client = httpx.Client(timeout=30.0) @@ -107,13 +107,26 @@ def _get_embedding(text: str) -> list: except: return [] def _fetch_and_index(url: str, category: str = ""): - """Fetch a URL through round-robin proxy, extract text, index into OpenSearch + Qdrant.""" + """Fetch URL through proxy, extract text + images, index into OpenSearch + Qdrant.""" proxy_name, proxy_url = _next_proxy() try: pc = httpx.Client(proxy=proxy_url, timeout=60.0) r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"}) if r.status_code != 200: return None html_text = r.text + # Extract image URLs + image_urls = [] + for m in re.finditer(r']+src=["\x27]([^"\x27]+)["\x27]', html_text, re.IGNORECASE): + src = m.group(1) + if src.startswith("//"): src = "https:" + src + elif src.startswith("/"): + base = urlparse(url) + src = f"{base.scheme}://{base.netloc}{src}" + elif not src.startswith("http"): continue + if any(x in src.lower() for x in ["icon","logo","pixel","tracking","analytics","1x1","spacer","blank","avatar","badge"]): continue + image_urls.append(src) + image_urls = list(dict.fromkeys(image_urls))[:10] + # Text extraction text = re.sub(r']*>.*?','',html_text,flags=re.DOTALL|re.IGNORECASE) text = re.sub(r']*>.*?','',text,flags=re.DOTALL|re.IGNORECASE) text = re.sub(r'<[^>]+>',' ',text) @@ -121,14 +134,14 @@ def _fetch_and_index(url: str, category: str = ""): text = html_mod.unescape(text) title_m = re.search(r']*>(.*?)',html_text,re.IGNORECASE|re.DOTALL) title = html_mod.unescape(title_m.group(1).strip()) if title_m else url - desc_m = re.search(r']+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE) + desc_m = re.search(r']+name=["\x27]description["\x27][^>]+content=["\x27]([^"\x27]+)',html_text,re.IGNORECASE) excerpt = desc_m.group(1)[:500] if desc_m else text[:500] domain = urlparse(url).netloc if not category: for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items(): if k in domain: category = v; break if not category: category = "web" - doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()} + doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"images":image_urls,"image_count":len(image_urls),"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()} _ensure_index() client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"}) emb = _get_embedding(excerpt[:1000]) @@ -136,7 +149,7 @@ def _fetch_and_index(url: str, category: str = ""): try: client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]}) except: pass - return {"title":title,"domain":domain,"category":category,"size":len(text)} + return {"title":title,"domain":domain,"category":category,"size":len(text),"images":len(image_urls),"proxy":proxy_name} except: return None @@ -383,6 +396,8 @@ def status(): @app.get("/api/search") def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10): + if not q or not q.strip(): + return {"query": "", "total": 0, "hits": [], "hint": "Empty query — provide a search term"} user = _auth(request) _track_usage(user["id"], "search_web", request) _ensure_index() @@ -394,7 +409,7 @@ def search_web(request: Request, q: str = Query(...), category: str = "", limit: hits = [] for h in result.get("hits",{}).get("hits",[]): src = h["_source"] - hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]}) + hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"],"images":src.get("images",[]),"image_count":src.get("image_count",0)}) return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits} except Exception as e: return {"query":q,"total":0,"hits":[],"note":str(e)} diff --git a/docker-compose.yml b/docker-compose.yml index b23710c..9e10c38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,46 @@ services: timeout: 5s retries: 3 + # ── Database ─────────────────────────────────────────────── + postgres: + image: postgres:16-alpine + container_name: postgres + environment: + - POSTGRES_USER=research + - POSTGRES_PASSWORD=ResearchDB2026! + - POSTGRES_DB=research_engine + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD", "pg_isready", "-U", "research"] + interval: 10s + timeout: 5s + retries: 5 + + # ── API + Dashboard ──────────────────────────────────────── + backend: + build: . + container_name: research-backend + network_mode: host + restart: unless-stopped + depends_on: + opensearch: + condition: service_healthy + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/"] + interval: 30s + timeout: 10s + retries: 3 + volumes: yacy_data: opensearch_data: redis_data: + postgres_data: