diff --git a/backend.py b/backend.py
index 99d31aa..4648d26 100644
--- a/backend.py
+++ b/backend.py
@@ -8,6 +8,7 @@ Exposes REST API consumed by the MCP proxy (MacBook) and dashboard.
import os
import json
import httpx
+import hashlib
from fastapi import FastAPI, Query, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
@@ -124,7 +125,7 @@ def search_web(q: str = Query(...), category: str = "", limit: int = 10):
if category:
body["query"]["bool"]["filter"] = [{"term": {"category": category}}]
try:
- r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body, params={"refresh": "true"})
+ r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body)
result = r.json()
hits = []
for h in result.get("hits", {}).get("hits", []):
@@ -156,8 +157,73 @@ def semantic_search(q: str = Query(...), limit: int = 10):
# ── Crawl ─────────────────────────────────────────────────────
+import re
+import html as html_mod
+
+def _fetch_and_index(url: str, category: str = ""):
+ """Fetch a URL, extract text, and index into OpenSearch immediately."""
+ try:
+ r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0)
+ if r.status_code != 200:
+ return None
+ html_text = r.text
+ # Basic HTML-to-text
+ text = re.sub(r'', '', html_text, flags=re.DOTALL|re.IGNORECASE)
+ text = re.sub(r'', '', text, flags=re.DOTALL|re.IGNORECASE)
+ text = re.sub(r'<[^>]+>', ' ', text)
+ text = re.sub(r'\s+', ' ', text).strip()
+ text = html_mod.unescape(text)
+ # Extract title
+ title_match = re.search(r'
]*>(.*?)', html_text, re.IGNORECASE|re.DOTALL)
+ title = html_mod.unescape(title_match.group(1).strip()) if title_match else url
+ # Extract meta description
+ desc_match = re.search(r']+name=["\']description["\'][^>]+content=["\']([^"\']+)', html_text, re.IGNORECASE)
+ excerpt = desc_match.group(1)[:500] if desc_match else text[:500]
+ # Derive domain + category
+ from urllib.parse import urlparse
+ domain = urlparse(url).netloc
+ if not category:
+ cat_map = {"wikipedia": "reference", "github": "software", "arxiv": "science",
+ "docs.": "documentation", "blog.": "blog", "news.": "news"}
+ for k, v in cat_map.items():
+ if k in domain:
+ category = v
+ break
+ if not category:
+ category = "web"
+ # Index into OpenSearch
+ import datetime
+ doc = {
+ "url": url, "title": title, "content": text[:50000],
+ "excerpt": excerpt[:1000], "category": category,
+ "source_domain": domain,
+ "crawled_at": datetime.datetime.utcnow().isoformat(),
+ "indexed_at": datetime.datetime.utcnow().isoformat(),
+ }
+ _ensure_index()
+ client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",
+ json=doc, params={"refresh": "true"})
+ # Also index into Qdrant
+ emb = _get_embedding(excerpt[:1000])
+ if emb:
+ try:
+ _ensure_qdrant()
+ client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points", json={
+ "points": [{"id": hashlib.md5(url.encode()).hexdigest(),
+ "vector": emb, "payload": {"url": url, "title": title, "excerpt": excerpt[:500]}}]
+ })
+ except Exception:
+ pass
+ return {"title": title, "domain": domain, "category": category, "size": len(text)}
+ except Exception as e:
+ return None
+
+
@app.get("/api/crawl")
def crawl_url(url: str = Query(...), depth: int = 1):
+ # 1. Fetch and index immediately into OpenSearch + Qdrant
+ indexed = _fetch_and_index(url)
+ # 2. Also submit to YaCy for deeper crawling
try:
r = client.get(f"{YACY_URL}/Crawler_p.json", params={
"crawlingDomMaxPages": 50, "crawlingDepth": depth,
@@ -166,9 +232,9 @@ def crawl_url(url: str = Query(...), depth: int = 1):
"indexText": "on", "indexMedia": "on",
"crawlingMode": "url", "cachePolicy": "iffresh",
})
- return {"status": "crawl_started", "url": url, "depth": depth}
+ return {"status": "crawl_started", "url": url, "depth": depth, "indexed": indexed}
except Exception as e:
- return {"status": "error", "url": url, "error": str(e)}
+ return {"status": "indexed_only", "url": url, "indexed": indexed, "yacy_error": str(e)}
@app.get("/api/crawl-topic")