From da75b635f4d57c06495971a5d17b88af6b99d6d5 Mon Sep 17 00:00:00 2001
From: drjones <drjones@drjoness-MacBook-Air.local>
Date: Tue, 4 Aug 2026 06:12:05 -0700
Subject: [PATCH] Add fetch-and-index pipeline: crawl immediately indexes into
 OpenSearch+Qdrant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- _fetch_and_index() fetches, strips HTML, extracts title/meta, indexes
- crawl_url now indexes immediately (not just submits to YaCy)
- Fixed search: removed invalid refresh param on _search requests
- Auto-categorization by domain (wikipedia→reference, github→software, etc)
- Verified: 5 docs in OpenSearch, 5 vectors in Qdrant
---
 backend.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 3 deletions(-)

diff --git a/backend.py b/backend.py
index 99d31aa..4648d26 100644
--- a/backend.py
+++ b/backend.py
@@ -8,6 +8,7 @@ Exposes REST API consumed by the MCP proxy (MacBook) and dashboard.
 import os
 import json
 import httpx
+import hashlib
 from fastapi import FastAPI, Query, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse
@@ -124,7 +125,7 @@ def search_web(q: str = Query(...), category: str = "", limit: int = 10):
     if category:
         body["query"]["bool"]["filter"] = [{"term": {"category": category}}]
     try:
-        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body, params={"refresh": "true"})
+        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body)
         result = r.json()
         hits = []
         for h in result.get("hits", {}).get("hits", []):
@@ -156,8 +157,73 @@ def semantic_search(q: str = Query(...), limit: int = 10):
 
 # ── Crawl ─────────────────────────────────────────────────────
 
+import re
+import html as html_mod
+
+def _fetch_and_index(url: str, category: str = ""):
+    """Fetch a URL, extract text, and index into OpenSearch immediately."""
+    try:
+        r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"}, timeout=15.0)
+        if r.status_code != 200:
+            return None
+        html_text = r.text
+        # Basic HTML-to-text
+        text = re.sub(r'<script[^>]*>.*?</script>', '', html_text, flags=re.DOTALL|re.IGNORECASE)
+        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL|re.IGNORECASE)
+        text = re.sub(r'<[^>]+>', ' ', text)
+        text = re.sub(r'\s+', ' ', text).strip()
+        text = html_mod.unescape(text)
+        # Extract title
+        title_match = re.search(r'<title[^>]*>(.*?)</title>', html_text, re.IGNORECASE|re.DOTALL)
+        title = html_mod.unescape(title_match.group(1).strip()) if title_match else url
+        # Extract meta description
+        desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)', html_text, re.IGNORECASE)
+        excerpt = desc_match.group(1)[:500] if desc_match else text[:500]
+        # Derive domain + category
+        from urllib.parse import urlparse
+        domain = urlparse(url).netloc
+        if not category:
+            cat_map = {"wikipedia": "reference", "github": "software", "arxiv": "science",
+                       "docs.": "documentation", "blog.": "blog", "news.": "news"}
+            for k, v in cat_map.items():
+                if k in domain:
+                    category = v
+                    break
+            if not category:
+                category = "web"
+        # Index into OpenSearch
+        import datetime
+        doc = {
+            "url": url, "title": title, "content": text[:50000],
+            "excerpt": excerpt[:1000], "category": category,
+            "source_domain": domain,
+            "crawled_at": datetime.datetime.utcnow().isoformat(),
+            "indexed_at": datetime.datetime.utcnow().isoformat(),
+        }
+        _ensure_index()
+        client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",
+                   json=doc, params={"refresh": "true"})
+        # Also index into Qdrant
+        emb = _get_embedding(excerpt[:1000])
+        if emb:
+            try:
+                _ensure_qdrant()
+                client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points", json={
+                    "points": [{"id": hashlib.md5(url.encode()).hexdigest(),
+                                "vector": emb, "payload": {"url": url, "title": title, "excerpt": excerpt[:500]}}]
+                })
+            except Exception:
+                pass
+        return {"title": title, "domain": domain, "category": category, "size": len(text)}
+    except Exception as e:
+        return None
+
+
 @app.get("/api/crawl")
 def crawl_url(url: str = Query(...), depth: int = 1):
+    # 1. Fetch and index immediately into OpenSearch + Qdrant
+    indexed = _fetch_and_index(url)
+    # 2. Also submit to YaCy for deeper crawling
     try:
         r = client.get(f"{YACY_URL}/Crawler_p.json", params={
             "crawlingDomMaxPages": 50, "crawlingDepth": depth,
@@ -166,9 +232,9 @@ def crawl_url(url: str = Query(...), depth: int = 1):
             "indexText": "on", "indexMedia": "on",
             "crawlingMode": "url", "cachePolicy": "iffresh",
         })
-        return {"status": "crawl_started", "url": url, "depth": depth}
+        return {"status": "crawl_started", "url": url, "depth": depth, "indexed": indexed}
     except Exception as e:
-        return {"status": "error", "url": url, "error": str(e)}
+        return {"status": "indexed_only", "url": url, "indexed": indexed, "yacy_error": str(e)}
 
 
 @app.get("/api/crawl-topic")