Commercial launch: Postgres auth, BTCPay payments, premium UI

- Postgres: users, api_keys, usage_log, payments tables
- Auth: API key system with rate limiting (402 when out of calls)
- BTCPay:  Bitcoin = 5 API calls, webhook for auto-credit
- Admin: sk-admin-unlimited-2026 with unlimited calls
- UI: Landing, Pricing (/5calls), Signup (no KYC), Dashboard with usage stats
- Footer: Created by drjones + Buy Me a Coffee link
- All endpoints auth-gated except signup, status, pages

Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-RUN pip install --no-cache-dir fastapi uvicorn httpx python-dotenv
+RUN pip install --no-cache-dir fastapi uvicorn httpx python-dotenv psycopg2-binary
 
 COPY backend.py .
 

backend.py

@@ -1,547 +1,517 @@
 #!/usr/bin/env python3
 """
-AI Research Engine — Backend API
+AI Research Engine — Commercial Backend
-Runs on CT 145, handles all heavy lifting.
+Postgres auth, BTCPay payments, API key system, premium UI.
-Exposes REST API consumed by the MCP proxy (MacBook) and dashboard.
 """
 
-import os
+import os, json, hashlib, re, html as html_mod, random, secrets, time
-import json
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
 import httpx
-import hashlib
+import psycopg2
-from fastapi import FastAPI, Query, HTTPException
+import psycopg2.extras
+from fastapi import FastAPI, Query, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 import uvicorn
 
-app = FastAPI(title="AI Research Engine Backend")
+app = FastAPI(title="AI Research Engine")
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
 
 # ── Config ──────────────────────────────────────────────────
-YACY_URL       = os.getenv("YACY_URL", "http://localhost:8090")
+YACY_URL       = os.getenv("YACY_URL", "http://host.docker.internal:8090")
-OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200")
+OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200")
 QDRANT_URL     = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
 OLLAMA_URL     = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434")
 OLLAMA_MODEL   = os.getenv("OLLAMA_MODEL", "ornith:latest")
 INDEX_NAME     = os.getenv("INDEX_NAME", "research_docs")
+BTCPAY_URL     = "https://10.30.20.140"
+BTCPAY_KEY     = "6026288e2e315984661c748baafd509e81a75f22"
+BTCPAY_STORE   = "8ERS6v1UyQ46bQqaWr4LotvMCJLsmUhH2sT8zbzfjkT6"
+WEBHOOK_SECRET = "XjfhDd9DzXsUkA91B4SwRz"
+PRICE_USD      = 5.00   # $5
+CALLS_PER_TIER = 5      # 5 API calls
+
+# Proxies
+PROXIES = [
+    "http://10.30.20.154:3128",  # Tokyo
+    "http://10.30.20.71:3128",   # London
+    "http://10.30.20.189:3128",  # Sydney
+]
+
+# DB
+def get_db():
+    return psycopg2.connect(
+        host="host.docker.internal", port=5432,
+        user="research", password="ResearchDB2026!",
+        database="research_engine"
+    )
 
 client = httpx.Client(timeout=30.0)
 ollama = httpx.Client(timeout=120.0, base_url=OLLAMA_URL)
 
-# ── Helpers ──────────────────────────────────────────────────
+# ── Auth Helpers ─────────────────────────────────────────────
+
+def _auth(request: Request) -> dict:
+    """Authenticate by API key, return user row or raise 401."""
+    api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
+    if not api_key:
+        raise HTTPException(401, "API key required. Get one at /signup or /pricing.")
+    db = get_db()
+    cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
+    user = cur.fetchone()
+    db.close()
+    if not user:
+        raise HTTPException(401, "Invalid API key.")
+    if not user["is_admin"] and user["calls_remaining"] <= 0:
+        raise HTTPException(402, "No calls remaining. Buy more at /pricing.")
+    return dict(user)
+
+def _track_usage(user_id: int, tool: str, request: Request):
+    """Deduct a call and log usage."""
+    db = get_db()
+    cur = db.cursor()
+    cur.execute("UPDATE users SET calls_remaining = calls_remaining - 1, total_calls = total_calls + 1, last_used_at = NOW() WHERE id = %s AND NOT is_admin", (user_id,))
+    cur.execute("INSERT INTO usage_log (user_id, tool_name, endpoint, ip_address) VALUES (%s,%s,%s,%s)",
+                (user_id, tool, str(request.url), request.client.host if request.client else ""))
+    db.commit()
+    db.close()
+
+# ── Search Helpers (unchanged from v1) ────────────────────────
 
 def _ensure_index():
-    try:
+    try: client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}")
-        client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}")
+    except: client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"url":{"type":"keyword"},"title":{"type":"text"},"content":{"type":"text"},"excerpt":{"type":"text"},"category":{"type":"keyword"},"source_domain":{"type":"keyword"},"crawled_at":{"type":"date"},"indexed_at":{"type":"date"},"metadata":{"type":"object"}}}})
-    except Exception:
-        try:
-            client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={
-                "settings": {"number_of_shards": 1, "number_of_replicas": 0},
-                "mappings": {"properties": {
-                    "url": {"type": "keyword"}, "title": {"type": "text"},
-                    "content": {"type": "text"}, "excerpt": {"type": "text"},
-                    "category": {"type": "keyword"}, "source_domain": {"type": "keyword"},
-                    "crawled_at": {"type": "date"}, "indexed_at": {"type": "date"},
-                    "metadata": {"type": "object"},
-                }}
-            })
-        except Exception:
-            pass
-
-def _ensure_qdrant():
-    try:
-        client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}")
-    except Exception:
-        try:
-            client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}", json={
-                "vectors": {"size": 768, "distance": "Cosine"}
-            })
-        except Exception:
-            pass
 
 def _ai_chat(prompt: str, system: str = "You are a research assistant. Be concise and factual.") -> str:
-    r = ollama.post("/api/chat", json={
+    r = ollama.post("/api/chat", json={"model":OLLAMA_MODEL,"messages":[{"role":"system","content":system},{"role":"user","content":prompt}],"stream":False,"options":{"temperature":0.3,"num_predict":2048}})
-        "model": OLLAMA_MODEL, "messages": [
-            {"role": "system", "content": system},
-            {"role": "user", "content": prompt},
-        ], "stream": False,
-        "options": {"temperature": 0.3, "num_predict": 2048},
-    })
     body = r.json()
-    return body.get("message", {}).get("content", "") or body.get("thinking", "") or ""
+    return body.get("message",{}).get("content","") or body.get("thinking","") or ""
 
 def _get_embedding(text: str) -> list:
     try:
-        r = ollama.post("/api/embeddings", json={"model": "nomic-embed-text-v2-moe:latest", "prompt": text[:2048]})
+        r = ollama.post("/api/embeddings", json={"model":"nomic-embed-text-v2-moe:latest","prompt":text[:2048]})
-        return r.json().get("embedding", [])
+        return r.json().get("embedding",[])
-    except Exception:
+    except: return []
-        return []
+
+def _fetch_and_index(url: str, category: str = ""):
+    proxy_url = random.choice(PROXIES)
+    try:
+        pc = httpx.Client(proxy=proxy_url, timeout=15.0)
+        r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"})
+        if r.status_code != 200: return None
+        html_text = r.text
+        text = re.sub(r'<script[^>]*>.*?</script>','',html_text,flags=re.DOTALL|re.IGNORECASE)
+        text = re.sub(r'<style[^>]*>.*?</style>','',text,flags=re.DOTALL|re.IGNORECASE)
+        text = re.sub(r'<[^>]+>',' ',text)
+        text = re.sub(r'\s+',' ',text).strip()
+        text = html_mod.unescape(text)
+        title_m = re.search(r'<title[^>]*>(.*?)</title>',html_text,re.IGNORECASE|re.DOTALL)
+        title = html_mod.unescape(title_m.group(1).strip()) if title_m else url
+        desc_m = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE)
+        excerpt = desc_m.group(1)[:500] if desc_m else text[:500]
+        domain = urlparse(url).netloc
+        if not category:
+            for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items():
+                if k in domain: category = v; break
+            if not category: category = "web"
+        doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
+        _ensure_index()
+        client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"})
+        emb = _get_embedding(excerpt[:1000])
+        if emb:
+            try:
+                client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]})
+            except: pass
+        return {"title":title,"domain":domain,"category":category,"size":len(text)}
+    except: return None
 
 
-# ── Status ────────────────────────────────────────────────────
+# ═══════════════════════════════════════════════════════════════
+#  PUBLIC PAGES
+# ═══════════════════════════════════════════════════════════════
+
+@app.get("/", response_class=HTMLResponse)
+def landing():
+    return HTMLResponse(LANDING_HTML)
+
+@app.get("/pricing", response_class=HTMLResponse)
+def pricing_page():
+    return HTMLResponse(PRICING_HTML)
+
+@app.get("/signup", response_class=HTMLResponse)
+def signup_page():
+    return HTMLResponse(SIGNUP_HTML)
+
+@app.get("/dashboard", response_class=HTMLResponse)
+def dashboard_page():
+    return HTMLResponse(DASHBOARD_HTML)
+
+
+# ═══════════════════════════════════════════════════════════════
+#  AUTH API
+# ═══════════════════════════════════════════════════════════════
+
+@app.post("/api/signup")
+async def api_signup(request: Request):
+    data = await request.json()
+    email = data.get("email","").strip().lower()
+    if not email or "@" not in email:
+        return JSONResponse({"error":"Valid email required"}, 400)
+    api_key = "sk-" + secrets.token_hex(24)
+    db = get_db()
+    cur = db.cursor()
+    try:
+        cur.execute("INSERT INTO users (email, password_hash, api_key, calls_remaining) VALUES (%s,%s,%s,0)", (email, "bcrypt_placeholder", api_key))
+        db.commit()
+    except psycopg2.errors.UniqueViolation:
+        db.rollback()
+        cur.execute("SELECT api_key FROM users WHERE email = %s", (email,))
+        api_key = cur.fetchone()[0]
+    db.close()
+    return {"email": email, "api_key": api_key, "message": "Signup successful. Purchase API calls at /pricing to start using the engine."}
+
+@app.get("/api/my-usage")
+async def my_usage(request: Request):
+    # Allow even with 0 calls — users need to see their balance
+    api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
+    if not api_key:
+        raise HTTPException(401, "API key required.")
+    db = get_db()
+    cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
+    user = cur.fetchone()
+    if not user:
+        db.close()
+        raise HTTPException(401, "Invalid API key.")
+    user = dict(user)
+    cur.execute("SELECT tool_name, COUNT(*) as cnt FROM usage_log WHERE user_id = %s GROUP BY tool_name ORDER BY cnt DESC", (user["id"],))
+    user["usage_by_tool"] = [dict(r) for r in cur.fetchall()]
+    db.close()
+    return user
+
+
+# ═══════════════════════════════════════════════════════════════
+#  BTCPAY INTEGRATION
+# ═══════════════════════════════════════════════════════════════
+
+@app.post("/api/create-invoice")
+async def create_invoice(request: Request):
+    """Create BTCPay invoice for $5 = 5 calls. Works even with 0 calls."""
+    api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
+    if not api_key:
+        raise HTTPException(401, "API key required.")
+    db = get_db()
+    cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
+    user = cur.fetchone()
+    if not user:
+        db.close()
+        raise HTTPException(401, "Invalid API key.")
+    user = dict(user)
+    try:
+        r = httpx.post(f"{BTCPAY_URL}/api/v1/stores/{BTCPAY_STORE}/invoices",
+            headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"},
+            json={
+                "amount": str(PRICE_USD),
+                "currency": "USD",
+                "metadata": {
+                    "user_id": user["id"],
+                    "user_email": user["email"],
+                    "calls": CALLS_PER_TIER,
+                    "orderId": f"research-{user['id']}-{int(time.time())}"
+                },
+                "checkout": {"redirectURL": f"http://10.30.20.249:8000/dashboard"}
+            },
+            verify=False, timeout=15.0
+        )
+        inv = r.json()
+        # Log pending payment
+        db = get_db()
+        cur = db.cursor()
+        cur.execute("INSERT INTO payments (user_id, btcpay_invoice_id, amount_usd, calls_purchased, status) VALUES (%s,%s,%s,%s,'pending')",
+                    (user["id"], inv["id"], PRICE_USD, CALLS_PER_TIER))
+        db.commit()
+        db.close()
+        return {"invoice_id": inv["id"], "checkout_url": inv["checkoutLink"], "amount": f"${PRICE_USD}", "calls": CALLS_PER_TIER}
+    except Exception as e:
+        return JSONResponse({"error": str(e)}, 500)
+
+@app.post("/webhook/btcpay")
+async def btcpay_webhook(request: Request):
+    """BTCPay calls this when invoice is settled."""
+    body = await request.json()
+    event_type = body.get("type", "")
+    invoice_id = body.get("invoiceId", "")
+    metadata = body.get("metadata", {})
+
+    # Handle nested metadata (BTCPay wraps it)
+    if isinstance(metadata, dict):
+        user_id = metadata.get("user_id")
+        calls = metadata.get("calls", CALLS_PER_TIER)
+    else:
+        user_id = None
+        calls = CALLS_PER_TIER
+
+    if event_type in ("InvoiceSettled", "InvoiceProcessing") and user_id:
+        db = get_db()
+        cur = db.cursor()
+        cur.execute("UPDATE payments SET status = 'settled', settled_at = NOW() WHERE btcpay_invoice_id = %s", (invoice_id,))
+        cur.execute("UPDATE users SET calls_remaining = calls_remaining + %s WHERE id = %s", (calls, user_id))
+        db.commit()
+        db.close()
 
-@app.get("/health")
-def health():
     return {"status": "ok"}
 
+
+# ═══════════════════════════════════════════════════════════════
+#  API TOOLS (auth-gated)
+# ═══════════════════════════════════════════════════════════════
+
 @app.get("/api/status")
 def status():
     svc = {}
     try: 
         r = client.get(f"{OPENSEARCH_URL}/_cluster/health")
-        cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json() if r.status_code == 200 else {"count": 0}
+        cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json()
-        svc["opensearch"] = {"status": r.json().get("status", "down"), "documents": cnt.get("count", 0)}
+        svc["opensearch"] = {"status": r.json().get("status"), "documents": cnt.get("count",0)}
-    except Exception:
+    except: svc["opensearch"] = {"status":"down"}
-        svc["opensearch"] = {"status": "down"}
     try:
-        r = client.get(f"{QDRANT_URL}/healthz")
+        client.get(f"{QDRANT_URL}/healthz")
         col = client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}").json()
-        svc["qdrant"] = {"status": "ok", "points": col.get("result", {}).get("points_count", 0)}
+        svc["qdrant"] = {"status":"ok","points":col.get("result",{}).get("points_count",0)}
-    except Exception:
+    except: svc["qdrant"] = {"status":"down"}
-        svc["qdrant"] = {"status": "down"}
     try:
-        r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": "test", "maximumRecords": 1})
+        client.get(f"{YACY_URL}/yacysearch.json",params={"query":"test","maximumRecords":1})
-        svc["yacy"] = {"status": "ok" if r.status_code == 200 else "down"}
+        svc["yacy"] = {"status":"ok"}
-    except Exception:
+    except: svc["yacy"] = {"status":"down"}
-        svc["yacy"] = {"status": "down"}
     try:
         r = ollama.get("/api/tags")
-        svc["ollama"] = {"status": "ok", "models": len(r.json().get("models", []))}
+        svc["ollama"] = {"status":"ok","models":len(r.json().get("models",[]))}
-    except Exception:
+    except: svc["ollama"] = {"status":"down"}
-        svc["ollama"] = {"status": "down"}
+    # DB stats
-    return {"services": svc}
+    try:
-
+        db = get_db()
-
+        cur = db.cursor()
-# ── Search ────────────────────────────────────────────────────
+        cur.execute("SELECT COUNT(*) FROM users")
+        users = cur.fetchone()[0]
+        cur.execute("SELECT COALESCE(SUM(total_calls),0) FROM users")
+        total = cur.fetchone()[0]
+        cur.execute("SELECT COUNT(*) FROM payments WHERE status = 'settled'")
+        payments = cur.fetchone()[0]
+        db.close()
+        svc["business"] = {"status":"ok","users":users,"total_api_calls":total,"payments_settled":payments}
+    except: svc["business"] = {"status":"down"}
+    return {"services":svc}
 
 @app.get("/api/search")
-def search_web(q: str = Query(...), category: str = "", limit: int = 10):
+def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10):
+    user = _auth(request)
+    _track_usage(user["id"], "search_web", request)
     _ensure_index()
-    body = {
+    body = {"size":limit,"query":{"bool":{"must":[{"multi_match":{"query":q,"fields":["title^3","content","excerpt"]}}]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":2}}}}
-        "size": limit,
+    if category: body["query"]["bool"]["filter"] = [{"term":{"category":category}}]
-        "query": {"bool": {"must": [{"multi_match": {"query": q, "fields": ["title^3", "content", "excerpt"]}}]}},
-        "highlight": {"fields": {"content": {"fragment_size": 200, "number_of_fragments": 2}}},
-    }
-    if category:
-        body["query"]["bool"]["filter"] = [{"term": {"category": category}}]
     try:
         r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body)
         result = r.json()
         hits = []
-        for h in result.get("hits", {}).get("hits", []):
+        for h in result.get("hits",{}).get("hits",[]):
             src = h["_source"]
-            hits.append({"url": src.get("url"), "title": src.get("title"),
+            hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]})
-                "excerpt": src.get("excerpt") or (h.get("highlight", {}).get("content", [""])[0]),
+        return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits}
-                "category": src.get("category"), "crawled_at": src.get("crawled_at"), "score": h["_score"]})
-        return {"query": q, "total": result.get("hits", {}).get("total", {}).get("value", 0), "hits": hits}
     except Exception as e:
-        return {"query": q, "total": 0, "hits": [], "note": f"Index may be empty. {e}"}
+        return {"query":q,"total":0,"hits":[],"note":str(e)}
-
 
 @app.get("/api/semantic-search")
-def semantic_search(q: str = Query(...), limit: int = 10):
+def semantic_search(request: Request, q: str = Query(...), limit: int = 10):
-    _ensure_qdrant()
+    user = _auth(request)
+    _track_usage(user["id"], "semantic_search", request)
     emb = _get_embedding(q)
-    if not emb:
+    if not emb: return {"hits":[],"error":"Embedding model unavailable"}
-        return {"hits": [], "error": "Embedding model not available"}
     try:
-        r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search", json={
+        r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":limit,"with_payload":True,"with_vector":False})
-            "vector": emb, "limit": limit, "with_payload": True, "with_vector": False})
+        hits = [{"url":p.get("payload",{}).get("url"),"title":p.get("payload",{}).get("title"),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:300],"score":p.get("score")} for p in r.json().get("result",[])]
-        hits = [{"url": p.get("payload", {}).get("url"), "title": p.get("payload", {}).get("title"),
+        return {"query":q,"total":len(hits),"hits":hits}
-            "excerpt": str(p.get("payload", {}).get("excerpt", ""))[:300], "score": p.get("score")}
-            for p in r.json().get("result", [])]
-        return {"query": q, "total": len(hits), "hits": hits}
     except Exception as e:
-        return {"hits": [], "error": str(e)}
+        return {"hits":[],"error":str(e)}
-
-
-# ── Crawl ─────────────────────────────────────────────────────
-
-import re
-import html as html_mod
-import random
-
-# NordVPN HTTP proxies for geo-distributed fetching
-PROXIES = [
-    "http://10.30.20.154:3128",  # Tokyo, Japan
-    "http://10.30.20.71:3128",   # London, UK
-    "http://10.30.20.189:3128",  # Sydney, Australia
-]
-
-def _fetch_and_index(url: str, category: str = ""):
-    """Fetch a URL, extract text, and index into OpenSearch immediately."""
-    proxy_url = random.choice(PROXIES)
-    try:
-        # Create a per-request client with proxy
-        proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0)
-        r = proxy_client.get(url,
-            headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"},
-        )
-        if r.status_code != 200:
-            return None
-        html_text = r.text
-        # Basic HTML-to-text
-        text = re.sub(r'<script[^>]*>.*?</script>', '', html_text, flags=re.DOTALL|re.IGNORECASE)
-        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL|re.IGNORECASE)
-        text = re.sub(r'<[^>]+>', ' ', text)
-        text = re.sub(r'\s+', ' ', text).strip()
-        text = html_mod.unescape(text)
-        # Extract title
-        title_match = re.search(r'<title[^>]*>(.*?)</title>', html_text, re.IGNORECASE|re.DOTALL)
-        title = html_mod.unescape(title_match.group(1).strip()) if title_match else url
-        # Extract meta description
-        desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)', html_text, re.IGNORECASE)
-        excerpt = desc_match.group(1)[:500] if desc_match else text[:500]
-        # Derive domain + category
-        from urllib.parse import urlparse
-        domain = urlparse(url).netloc
-        if not category:
-            cat_map = {"wikipedia": "reference", "github": "software", "arxiv": "science",
-                       "docs.": "documentation", "blog.": "blog", "news.": "news"}
-            for k, v in cat_map.items():
-                if k in domain:
-                    category = v
-                    break
-            if not category:
-                category = "web"
-        # Index into OpenSearch
-        import datetime
-        doc = {
-            "url": url, "title": title, "content": text[:50000],
-            "excerpt": excerpt[:1000], "category": category,
-            "source_domain": domain,
-            "crawled_at": datetime.datetime.utcnow().isoformat(),
-            "indexed_at": datetime.datetime.utcnow().isoformat(),
-        }
-        _ensure_index()
-        client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",
-                   json=doc, params={"refresh": "true"})
-        # Also index into Qdrant
-        emb = _get_embedding(excerpt[:1000])
-        if emb:
-            try:
-                _ensure_qdrant()
-                client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points", json={
-                    "points": [{"id": hashlib.md5(url.encode()).hexdigest(),
-                                "vector": emb, "payload": {"url": url, "title": title, "excerpt": excerpt[:500]}}]
-                })
-            except Exception:
-                pass
-        return {"title": title, "domain": domain, "category": category, "size": len(text)}
-    except Exception as e:
-        return None
-
 
 @app.get("/api/crawl")
-def crawl_url(url: str = Query(...), depth: int = 1):
+def crawl_url(request: Request, url: str = Query(...), depth: int = 1):
-    # 1. Fetch and index immediately into OpenSearch + Qdrant
+    user = _auth(request)
+    _track_usage(user["id"], "crawl_url", request)
     indexed = _fetch_and_index(url)
-    # 2. Also submit to YaCy for deeper crawling
     try:
-        r = client.get(f"{YACY_URL}/Crawler_p.json", params={
+        client.get(f"{YACY_URL}/Crawler_p.json", params={"crawlingDomMaxPages":50,"crawlingDepth":depth,"crawlingStart":url,"crawlingQ":"on","bookmarkTitle":"research","bookmarkFolder":"/research","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"})
-            "crawlingDomMaxPages": 50, "crawlingDepth": depth,
+        return {"status":"crawl_started","url":url,"depth":depth,"indexed":indexed}
-            "crawlingStart": url, "crawlingQ": "on",
+    except:
-            "bookmarkTitle": "research", "bookmarkFolder": "/research",
+        return {"status":"indexed_only","url":url,"indexed":indexed}
-            "indexText": "on", "indexMedia": "on",
-            "crawlingMode": "url", "cachePolicy": "iffresh",
-        })
-        return {"status": "crawl_started", "url": url, "depth": depth, "indexed": indexed}
-    except Exception as e:
-        return {"status": "indexed_only", "url": url, "indexed": indexed, "yacy_error": str(e)}
 
-
+@app.get("/api/research")
-@app.get("/api/crawl-topic")
+def research_topic(request: Request, topic: str = Query(...)):
-def crawl_topic(topic: str = Query(...), max_urls: int = 20):
+    user = _auth(request)
-    discovered = []
+    _track_usage(user["id"], "research_topic", request)
+    steps = []; kw_result = {"hits":[]}; sem_result = {"hits":[]}
     try:
-        r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": topic, "maximumRecords": max_urls, "resource": "global"})
+        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":5,"query":{"multi_match":{"query":topic,"fields":["title^3","content","excerpt"]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":1}}}})
-        for ch in r.json().get("channels", []):
+        hits = [{"title":h["_source"].get("title"),"excerpt":h.get("highlight",{}).get("content",[h["_source"].get("excerpt","")])[0]} for h in r.json().get("hits",{}).get("hits",[])]
-            for item in ch.get("items", []):
+        kw_result = {"total":len(hits),"hits":hits}; steps.append("keyword_search_done")
-                if item.get("link"):
+    except: steps.append("keyword_search_skipped")
-                    discovered.append({"url": item["link"], "title": item.get("title", ""), "snippet": item.get("description", "")})
-    except Exception as e:
-        return {"topic": topic, "error": f"Discovery failed: {e}", "urls_discovered": 0, "urls_crawled": 0}
-
-    crawled = 0
-    for u in discovered[:max_urls]:
     try:
-            client.get(f"{YACY_URL}/Crawler_p.json", params={
+        emb = _get_embedding(topic)
-                "crawlingDomMaxPages": 10, "crawlingDepth": 0,
+        if emb:
-                "crawlingStart": u["url"], "crawlingQ": "on",
+            r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":5,"with_payload":True})
-                "indexText": "on", "indexMedia": "on",
+            hits2 = [{"title":p.get("payload",{}).get("title",""),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:200],"score":p.get("score")} for p in r.json().get("result",[])]
-                "crawlingMode": "url", "cachePolicy": "iffresh",
+            sem_result = {"total":len(hits2),"hits":hits2}; steps.append("semantic_search_done")
-            }, timeout=5.0)
+    except: steps.append("semantic_search_skipped")
-            crawled += 1
-        except Exception:
-            pass
-
-    return {"topic": topic, "urls_discovered": len(discovered), "urls_crawled": crawled}
-
-
-# ── Document Retrieval ────────────────────────────────────────
-
-@app.get("/api/document")
-def retrieve_document(url: str = Query(...)):
     try:
-        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
+        r = client.get(f"{YACY_URL}/yacysearch.json",params={"query":topic,"maximumRecords":10,"resource":"global"})
-            "size": 1, "query": {"term": {"url": url}}})
+        discovered = [item.get("link") for ch in r.json().get("channels",[]) for item in ch.get("items",[]) if item.get("link")]
-        hits = r.json().get("hits", {}).get("hits", [])
+        for url in discovered[:10]:
-        if not hits:
+            try: client.get(f"{YACY_URL}/Crawler_p.json",params={"crawlingDomMaxPages":10,"crawlingDepth":0,"crawlingStart":url,"crawlingQ":"on","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"},timeout=5.0)
-            return {"error": "Not found", "url": url}
+            except: pass
-        src = hits[0]["_source"]
+        steps.append(f"crawl_dispatched_{len(discovered[:10])}")
-        return {"url": src.get("url"), "title": src.get("title"),
+    except: steps.append("crawl_skipped")
-            "content": src.get("content", "")[:10000], "excerpt": src.get("excerpt"),
+    all_src = ""
-            "category": src.get("category"), "crawled_at": src.get("crawled_at")}
+    for h in kw_result.get("hits",[])[:3] + sem_result.get("hits",[])[:3]:
-    except Exception as e:
+        all_src += f"- {h.get('title','Unknown')}: {h.get('excerpt','')[:200]}\n"
-        return {"error": str(e), "url": url}
+    summary = _ai_chat(f"Research topic: {topic}\n\nSources:\n{all_src}\n\nConcise research summary (3-5 paragraphs): key findings, important sources, knowledge gaps, next steps.") if all_src else ""
-
+    return {"topic":topic,"steps":steps,"keyword_results":kw_result,"semantic_results":sem_result,"ai_summary":summary}
-
-# ── AI Synthesis ──────────────────────────────────────────────
-
-@app.get("/api/summarize")
-def summarize_sources(urls: str = Query(...), instruction: str = "Summarize key points"):
-    url_list = [u.strip() for u in urls.split(",") if u.strip()]
-    combined = ""
-    for url in url_list[:5]:
-        try:
-            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
-            hits = r.json().get("hits", {}).get("hits", [])
-            if hits:
-                src = hits[0]["_source"]
-                combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content', src.get('excerpt',''))[:2000]}"
-        except Exception:
-            pass
-    if not combined.strip():
-        return {"error": "No content retrieved"}
-    summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.")
-    return {"instruction": instruction, "sources": len(url_list), "summary": summary}
-
-
-@app.get("/api/extract")
-def extract_information(url: str = Query(...), schema: str = Query("company names, products, prices")):
-    try:
-        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
-        hits = r.json().get("hits", {}).get("hits", [])
-        if not hits:
-            return {"error": "Not found", "url": url}
-        content = hits[0]["_source"].get("content", "")[:8000]
-        prompt = f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON."
-        result = _ai_chat(prompt, system="Extract structured data. Return ONLY valid JSON. No explanation.")
-        return {"url": url, "schema": schema, "extracted": result}
-    except Exception as e:
-        return {"error": str(e)}
-
 
 @app.get("/api/report")
-def create_report(topic: str = Query(...), sources: str = ""):
+def create_report(request: Request, topic: str = Query(...), sources: str = ""):
+    user = _auth(request)
+    _track_usage(user["id"], "create_report", request)
     if sources:
         urls = [u.strip() for u in sources.split(",") if u.strip()]
     else:
         try:
-            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
+            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":8,"query":{"multi_match":{"query":topic,"fields":["title^3","content"]}}})
-                "size": 8, "query": {"multi_match": {"query": topic, "fields": ["title^3", "content"]}}})
+            urls = [h["_source"]["url"] for h in r.json().get("hits",{}).get("hits",[])]
-            urls = [h["_source"]["url"] for h in r.json().get("hits", {}).get("hits", [])]
+        except: urls = []
-        except Exception:
-            urls = []
-
     gathered = ""
     for url in urls[:8]:
         try:
-            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
+            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
-            hits = r.json().get("hits", {}).get("hits", [])
+            hits = r.json().get("hits",{}).get("hits",[])
             if hits:
                 src = hits[0]["_source"]
                 gathered += f"\n\n### {src.get('title','Source')}\nURL: {url}\n{src.get('content',src.get('excerpt',''))[:1500]}"
-        except Exception:
+        except: pass
-            pass
+    report = _ai_chat(f"Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}\n\nGenerate a comprehensive report:\n1. Executive Summary\n2. Key Findings (numbered)\n3. Source Analysis\n4. Knowledge Gaps\n5. Recommendations\n\nBe thorough, use markdown, cite sources.", system="You are a senior research analyst. Produce thorough, structured reports.")
-
+    return {"topic":topic,"sources_used":len(urls),"report":report}
-    prompt = f"""Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}
-
-Generate a comprehensive report:
-1. Executive Summary
-2. Key Findings (numbered)
-3. Source Analysis
-4. Knowledge Gaps
-5. Recommendations
-
-Be thorough, use markdown, cite sources."""
-    report = _ai_chat(prompt, system="You are a senior research analyst. Produce thorough, structured reports.")
-    return {"topic": topic, "sources_used": len(urls), "report": report}
-
-
-# ── Research Pipeline ─────────────────────────────────────────
-
-@app.get("/api/research")
-def research_topic(topic: str = Query(...)):
-    steps = []
-    kw_result = {"hits": []}
-    sem_result = {"hits": []}
 
+@app.get("/api/document")
+def retrieve_document(request: Request, url: str = Query(...)):
+    user = _auth(request)
+    _track_usage(user["id"], "retrieve_document", request)
     try:
-        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
+        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
-            "size": 5, "query": {"multi_match": {"query": topic, "fields": ["title^3", "content", "excerpt"]}},
+        hits = r.json().get("hits",{}).get("hits",[])
-            "highlight": {"fields": {"content": {"fragment_size": 200, "number_of_fragments": 1}}},
+        if not hits: return {"error":"Not found","url":url}
-        })
+        src = hits[0]["_source"]
-        hits = [{"title": h["_source"].get("title"), "excerpt": h.get("highlight", {}).get("content", [h["_source"].get("excerpt", "")])[0]}
+        return {"url":src.get("url"),"title":src.get("title"),"content":src.get("content","")[:10000],"excerpt":src.get("excerpt"),"category":src.get("category"),"crawled_at":src.get("crawled_at")}
-                for h in r.json().get("hits", {}).get("hits", [])]
+    except Exception as e:
-        kw_result = {"total": len(hits), "hits": hits}
+        return {"error":str(e),"url":url}
-        steps.append("keyword_search_done")
-    except Exception:
-        steps.append("keyword_search_skipped")
 
+@app.get("/api/summarize")
+def summarize_sources(request: Request, urls: str = Query(...), instruction: str = "Summarize key points"):
+    user = _auth(request)
+    _track_usage(user["id"], "summarize_sources", request)
+    url_list = [u.strip() for u in urls.split(",") if u.strip()]
+    combined = ""
+    for url in url_list[:5]:
         try:
-        emb = _get_embedding(topic)
+            r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
-        if emb:
+            hits = r.json().get("hits",{}).get("hits",[])
-            r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search", json={
+            if hits:
-                "vector": emb, "limit": 5, "with_payload": True})
+                src = hits[0]["_source"]
-            hits2 = [{"title": p.get("payload", {}).get("title", ""), "excerpt": str(p.get("payload", {}).get("excerpt", ""))[:200],
+                combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content',src.get('excerpt',''))[:2000]}"
-                "score": p.get("score")} for p in r.json().get("result", [])]
+        except: pass
-            sem_result = {"total": len(hits2), "hits": hits2}
+    if not combined.strip(): return {"error":"No content retrieved"}
-            steps.append("semantic_search_done")
+    summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.")
-    except Exception:
+    return {"instruction":instruction,"sources":len(url_list),"summary":summary}
-        steps.append("semantic_search_skipped")
 
+@app.get("/api/extract")
+def extract_information(request: Request, url: str = Query(...), schema: str = Query("company names, products, prices")):
+    user = _auth(request)
+    _track_usage(user["id"], "extract_information", request)
     try:
-        r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": topic, "maximumRecords": 10, "resource": "global"})
+        r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
-        discovered = [item.get("link") for ch in r.json().get("channels", []) for item in ch.get("items", []) if item.get("link")]
+        hits = r.json().get("hits",{}).get("hits",[])
-        for url in discovered[:10]:
+        if not hits: return {"error":"Not found","url":url}
-            try:
+        content = hits[0]["_source"].get("content","")[:8000]
-                client.get(f"{YACY_URL}/Crawler_p.json", params={
+        result = _ai_chat(f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON.", system="Extract structured data. Return ONLY valid JSON.")
-                    "crawlingDomMaxPages": 10, "crawlingDepth": 0, "crawlingStart": url,
+        return {"url":url,"schema":schema,"extracted":result}
-                    "crawlingQ": "on", "indexText": "on", "indexMedia": "on",
+    except Exception as e:
-                    "crawlingMode": "url", "cachePolicy": "iffresh"}, timeout=5.0)
+        return {"error":str(e)}
-            except Exception:
-                pass
-        steps.append(f"crawl_dispatched_{len(discovered[:10])}")
-    except Exception:
-        steps.append("crawl_skipped")
-
-    all_sources = ""
-    for h in kw_result.get("hits", [])[:3] + sem_result.get("hits", [])[:3]:
-        all_sources += f"- {h.get('title', 'Unknown')}: {h.get('excerpt', '')[:200]}\n"
-
-    summary = ""
-    if all_sources:
-        summary = _ai_chat(
-            f"Research topic: {topic}\n\nSources:\n{all_sources}\n\nConcise research summary (3-5 paragraphs): "
-            "key findings, important sources, knowledge gaps, next steps.")
-
-    return {"topic": topic, "steps": steps,
-        "keyword_results": kw_result, "semantic_results": sem_result,
-        "ai_summary": summary}
 
 
-# ── Dashboard ─────────────────────────────────────────────────
+# ═══════════════════════════════════════════════════════════════
+#  UI TEMPLATES
+# ═══════════════════════════════════════════════════════════════
 
-@app.get("/", response_class=HTMLResponse)
+CSS = """
-def dashboard():
+:root{--bg:#050508;--surface:#0c0c14;--border:#1e1e30;--accent:#7c3aed;--accent2:#a855f7;--gold:#f59e0b;--text:#f4f4f5;--muted:#a1a1aa;--green:#22c55e;--red:#ef4444}
-    return HTMLResponse("""
-<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
-<title>AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet">
-<style>
-:root{--bg:#09090b;--surface:#18181b;--border:#27272a;--accent:#7c3aed;--accent2:#a855f7;--text:#f4f4f5;--muted:#a1a1aa;--green:#22c55e;--red:#ef4444;--amber:#f59e0b}
 *{box-sizing:border-box;margin:0;padding:0}
-body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh}
+body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh;overflow-x:hidden}
-.header{background:var(--surface);border-bottom:1px solid var(--border);padding:20px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:10}
+.gradient-bg{background:radial-gradient(ellipse 80% 50% at 50% -20%,rgba(120,60,255,0.15),transparent)}
-.header h1{font-size:1.4rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--accent2),#ec4899);-webkit-background-clip:text;-webkit-text-fill-color:transparent;letter-spacing:-0.02em}
+.header{background:rgba(12,12,20,0.8);backdrop-filter:blur(20px);border-bottom:1px solid var(--border);padding:16px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:100}
-.header .badge{font-size:0.75rem;color:var(--muted);background:var(--border);padding:6px 12px;border-radius:999px}
+.header h1{font-size:1.3rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--accent2),#ec4899);-webkit-background-clip:text;-webkit-text-fill-color:transparent}
-.container{max-width:1400px;margin:0 auto;padding:32px 24px}
+.header nav{display:flex;gap:20px;align-items:center}
-.hero{text-align:center;padding:48px 0 32px}
+.header nav a{color:var(--muted);text-decoration:none;font-size:0.9rem;transition:color .2s}
-.hero h2{font-size:2.2rem;font-weight:800;letter-spacing:-0.03em;margin-bottom:12px}
+.header nav a:hover{color:var(--text)}
-.hero p{color:var(--muted);font-size:1.1rem;max-width:600px;margin:0 auto}
+.btn{padding:12px 24px;border-radius:12px;border:none;font-weight:600;font-size:0.95rem;cursor:pointer;transition:all .2s;text-decoration:none;display:inline-block}
-.search-box{display:flex;gap:12px;max-width:800px;margin:0 auto 40px}
-.search-box input{flex:1;padding:16px 24px;border-radius:16px;border:1px solid var(--border);background:var(--surface);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s}
-.search-box input:focus{border-color:var(--accent)}
-.search-box button{padding:16px 32px;border-radius:16px;border:none;font-weight:600;font-size:1rem;cursor:pointer;transition:all .2s}
 .btn-primary{background:var(--accent);color:white}
-.btn-primary:hover{background:var(--accent2)}
+.btn-primary:hover{background:var(--accent2);transform:translateY(-1px);box-shadow:0 8px 25px rgba(124,58,237,0.3)}
-.btn-secondary{background:var(--surface);color:var(--text);border:1px solid var(--border)}
+.btn-gold{background:linear-gradient(135deg,var(--gold),#fbbf24);color:#1a1a1a}
-.btn-secondary:hover{border-color:var(--accent)}
+.btn-gold:hover{transform:translateY(-1px);box-shadow:0 8px 25px rgba(245,158,11,0.3)}
-.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:16px;margin-bottom:32px}
+.btn-outline{background:transparent;border:1px solid var(--border);color:var(--text)}
-.card{background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:24px;transition:border-color .2s}
+.container{max-width:1200px;margin:0 auto;padding:40px 24px}
-.card:hover{border-color:var(--accent)}
+.hero{text-align:center;padding:80px 0 60px}
-.card .label{color:var(--muted);font-size:0.8rem;text-transform:uppercase;letter-spacing:0.05em;margin-bottom:8px}
+.hero h2{font-size:3rem;font-weight:900;letter-spacing:-0.04em;line-height:1.1;margin-bottom:20px;background:linear-gradient(135deg,var(--text),var(--accent2));-webkit-background-clip:text;-webkit-text-fill-color:transparent}
-.card .value{font-size:1.8rem;font-weight:700}
+.hero p{color:var(--muted);font-size:1.2rem;max-width:650px;margin:0 auto 32px;line-height:1.6}
-.card .sub{color:var(--muted);font-size:0.85rem;margin-top:4px}
+.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:20px;margin:40px 0}
-.ok{color:var(--green)}.down{color:var(--red)}.warn{color:var(--amber)}
+.card{background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:28px;transition:all .2s}
-.action-bar{display:flex;gap:10px;margin-bottom:28px;flex-wrap:wrap}
+.card:hover{border-color:var(--accent);transform:translateY(-2px)}
-.action-bar button{padding:10px 20px;border-radius:10px;border:1px solid var(--border);background:var(--surface);color:var(--text);cursor:pointer;font-size:0.9rem;transition:all .15s}
+.card h3{font-size:1.2rem;margin-bottom:10px}
-.action-bar button:hover{border-color:var(--accent);background:#1f1f23}
+.card p{color:var(--muted);font-size:0.9rem;line-height:1.5}
-.results{display:flex;flex-direction:column;gap:12px}
+.price{font-size:3rem;font-weight:900;text-align:center;margin:20px 0}
-.result-card{background:var(--surface);border:1px solid var(--border);border-radius:14px;padding:20px;transition:border-color .15s}
+.price span{font-size:1rem;color:var(--muted);font-weight:400}
-.result-card:hover{border-color:var(--accent)}
+.feature-list{list-style:none;margin:20px 0}
-.result-card h3{margin-bottom:6px;font-size:1.1rem}
+.feature-list li{padding:8px 0;color:var(--muted);font-size:0.9rem}
-.result-card h3 a{color:var(--accent2);text-decoration:none}
+.feature-list li::before{content:'✓ ';color:var(--green);font-weight:700}
-.result-card h3 a:hover{text-decoration:underline}
+pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:16px;overflow-x:auto;font-size:0.85rem;margin:10px 0}
-.result-card .excerpt{color:var(--muted);font-size:0.9rem;line-height:1.5}
+.footer{text-align:center;padding:40px;color:var(--muted);font-size:0.85rem;border-top:1px solid var(--border);margin-top:60px}
-.result-card .meta{color:#71717a;font-size:0.78rem;margin-top:10px;display:flex;gap:16px}
+.footer a{color:var(--accent2);text-decoration:none}
-.loading{text-align:center;padding:48px;color:var(--muted)}
+input,textarea{width:100%;padding:14px 18px;border-radius:12px;border:1px solid var(--border);background:var(--bg);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s;margin-bottom:12px}
-.hidden{display:none}
+input:focus,textarea:focus{border-color:var(--accent)}
-pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:20px;overflow-x:auto;font-size:0.85rem;line-height:1.6;white-space:pre-wrap}
+"""
-.report{background:var(--surface);border:1px solid var(--border);border-radius:14px;padding:24px;line-height:1.7}
+
-.report h1,.report h2,.report h3{color:var(--accent2);margin:16px 0 8px}
+LANDING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>AI Research Engine — Private Knowledge Cloud</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}.glow{{position:absolute;width:600px;height:600px;border-radius:50%;filter:blur(120px);opacity:0.12;pointer-events:none}}.glow-1{{background:var(--accent);top:-200px;left:-100px}}.glow-2{{background:#ec4899;bottom:-200px;right:-100px}}</style></head><body class="gradient-bg"><div class="glow glow-1"></div><div class="glow glow-2"></div><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>The Ultimate Source of<br>Uncensored Internet Knowledge</h2><p>A self-hosted AI workforce that continuously gathers, organizes, and acts on information. Your private research cloud — no filters, no tracking, no limits.</p><div style="display:flex;gap:16px;justify-content:center;flex-wrap:wrap"><a href="/signup" class="btn btn-primary" style="font-size:1.1rem;padding:16px 36px">Get Started — Free Signup</a><a href="/pricing" class="btn btn-gold" style="font-size:1.1rem;padding:16px 36px">$5 Bitcoin = 5 API Calls ⚡</a></div></div><div class="grid"><div class="card"><h3>🔍 Deep Search</h3><p>Full-text search across millions of indexed documents. Find anything — no Google bubble, no censorship.</p></div><div class="card"><h3>🧬 Semantic Understanding</h3><p>Search by meaning, not keywords. Our vector engine finds conceptually related content even when the words differ.</p></div><div class="card"><h3>🕷️ Autonomous Crawling</h3><p>Point it at a topic and it discovers, crawls, and indexes every relevant source. Never miss a thing.</p></div><div class="card"><h3>🤖 AI Synthesis</h3><p>Local LLMs summarize, extract, and report. Your research, analyzed by AI running on our hardware.</p></div><div class="card"><h3>🌍 Geo-Distributed</h3><p>Crawls route through proxies in Tokyo, London, and Sydney. Bypass regional blocks automatically.</p></div><div class="card"><h3>🔐 Private & Self-Hosted</h3><p>Everything runs on our metal. No third-party APIs, no data harvesting, no surveillance.</p></div></div><div style="text-align:center;padding:40px 0"><h3 style="font-size:1.5rem;margin-bottom:16px">Trusted by AI Agents Worldwide</h3><p style="color:var(--muted);max-width:600px;margin:0 auto">10 MCP tools. One API key. Infinite knowledge. Agents connect and research autonomously — no browser needed.</p></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p><p style="margin-top:8px">AI Research Engine v2.0 · Self-hosted · No KYC · Bitcoin Only</p></div></div></body></html>"""
-.report ul,.report ol{padding-left:20px;margin:8px 0}
+
-.report li{margin:4px 0}
+PRICING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Pricing — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Simple Bitcoin Pricing</h2><p>No subscriptions. No KYC. Pay with Bitcoin, get API calls.</p></div><div style="max-width:500px;margin:0 auto"><div class="card" style="text-align:center;border-color:var(--accent);border-width:2px"><h3>🚀 Researcher Tier</h3><div class="price">$5<span> USD</span></div><p style="color:var(--muted);margin-bottom:16px">in Bitcoin · Lightning ⚡</p><ul class="feature-list" style="text-align:left;max-width:300px;margin:0 auto"><li>5 API calls</li><li>Full access to all 10 tools</li><li>Search, crawl, research, report</li><li>No expiration</li><li>No KYC required</li></ul><div style="margin-top:24px"><p style="color:var(--muted);font-size:0.85rem;margin-bottom:12px">👇 Sign up first, then buy from your dashboard</p><a href="/signup" class="btn btn-gold" style="width:100%">Sign Up & Get Your API Key</a></div></div></div><div class="grid" style="max-width:800px;margin:40px auto"><div class="card"><h3>⚡ Lightning Fast</h3><p>Payments settle in seconds. Your API calls are credited instantly.</p></div><div class="card"><h3>🔑 Bring Your Own Key</h3><p>Use your API key in any MCP client, script, or AI agent.</p></div><div class="card"><h3>📊 Track Usage</h3><p>Real-time dashboard shows your remaining calls and history.</p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p></div></div></body></html>"""
-</style></head><body>
+
-<div class="header"><h1>🧠 AI Research Engine</h1><div class="badge">Self-hosted · Private</div></div>
+SIGNUP_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Sign Up — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Get Your API Key</h2><p>No KYC. Just an email. Your key is generated instantly.</p></div><div class="card" style="max-width:500px;margin:0 auto"><input type="email" id="email" placeholder="you@email.com" style="font-size:1.1rem"><button class="btn btn-primary" onclick="signup()" style="width:100%;font-size:1.1rem">Generate My API Key</button><div id="result" style="margin-top:20px;display:none"><p style="color:var(--green);margin-bottom:8px">✅ Your API key is ready:</p><pre id="apikey" style="cursor:pointer;word-break:break-all" onclick="copyKey()"></pre><p style="color:var(--muted);font-size:0.85rem;margin-top:8px">Click key to copy. Store it safely. No password reset — it's yours.</p><p style="color:var(--muted);font-size:0.85rem">Next: <a href="/pricing" style="color:var(--accent2)">buy API calls</a> or <a href="/dashboard" style="color:var(--accent2)">go to dashboard</a></p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
-<div class="container">
+async function signup(){{const e=document.getElementById('email').value;if(!e)return;const r=await fetch('/api/signup',{{method:'POST',headers:{{'Content-Type':'application/json'}},body:JSON.stringify({{email:e}})}});const d=await r.json();document.getElementById('result').style.display='block';document.getElementById('apikey').textContent=d.api_key||d.error}}
-<div class="hero"><h2>Your Private Research Cloud</h2><p>Discover, index, and synthesize knowledge — all on your own infrastructure.</p></div>
+function copyKey(){{const t=document.getElementById('apikey').textContent;navigator.clipboard.writeText(t);alert('API key copied!')}}
-<div class="search-box">
+</script></body></html>"""
-<input type="text" id="query" placeholder="Research anything..." onkeydown="if(event.key==='Enter')search()">
+
-<button class="btn-primary" onclick="search()">🔍 Search</button>
+DASHBOARD_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Dashboard — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero" style="padding:40px 0"><h2>Dashboard</h2><p>Your account, usage, and API access.</p></div><div id="login-section"><div class="card" style="max-width:500px;margin:0 auto"><h3>🔑 Enter Your API Key</h3><input type="text" id="apikey-input" placeholder="sk-..." style="font-family:monospace"><button class="btn btn-primary" onclick="login()" style="width:100%">View My Dashboard</button></div></div><div id="dashboard-section" style="display:none"><div class="grid" id="stats"></div><div class="card" style="margin-bottom:20px"><h3>⚡ Buy API Calls</h3><p style="color:var(--muted);margin-bottom:16px">$5 Bitcoin = 5 API calls. Pay with Lightning.</p><button class="btn btn-gold" onclick="buyCalls()">Pay $5 with Bitcoin ⚡</button><div id="invoice-result" style="margin-top:16px"></div></div><div class="card"><h3>📋 Quick Reference</h3><pre style="font-size:0.8rem"># Use your key in any HTTP request:
-<button class="btn-secondary" onclick="research()" style="background:var(--accent);color:white;">🧪 Deep Research</button>
+curl -H "X-API-Key: YOUR_KEY" \\
-</div>
+  "http://10.30.20.249:8000/api/search?q=your+query"
-<div class="action-bar">
+
-<button onclick="loadStatus()">📡 System Status</button>
+# Or as a query parameter:
-<button onclick="toggleCrawl()">🕷️ Crawl URL</button>
+curl "http://10.30.20.249:8000/api/search?q=test&api_key=YOUR_KEY"</pre></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
-<button onclick="loadReport()">📄 Generate Report</button>
+let apiKey='';
-</div>
+async function login(){{apiKey=document.getElementById('apikey-input').value;if(!apiKey)return;loadDashboard()}}
-<div id="crawl-box" class="hidden" style="margin-bottom:20px;display:flex;gap:10px;">
+async function loadDashboard(){{try{{const r=await fetch('/api/my-usage?api_key='+apiKey);const d=await r.json();document.getElementById('login-section').style.display='none';document.getElementById('dashboard-section').style.display='block';let h='<div class="card"><h3>📧 '+d.email+'</h3><div class="price" style="font-size:2rem">'+d.calls_remaining+'<span> calls left</span></div><p style="color:var(--muted)">'+d.total_calls+' total · since '+d.created_at+'</p></div>';h+='<div class="card"><h3>🔑 API Key</h3><pre style="word-break:break-all;cursor:pointer" onclick="navigator.clipboard.writeText(\''+d.api_key+'\')">'+d.api_key+'</pre><p style="color:var(--muted);font-size:0.8rem">Click to copy</p></div>';if(d.usage_by_tool){{h+='<div class="card"><h3>📊 Usage by Tool</h3>';for(const t of d.usage_by_tool)h+='<p style="display:flex;justify-content:space-between"><span>'+t.tool_name+'</span><span style="color:var(--accent2)">'+t.cnt+' calls</span></p>';h+='</div>'}}if(d.is_admin)h+='<div class="card" style="border-color:var(--gold)"><h3>👑 Admin Account</h3><p style="color:var(--gold)">Unlimited calls · Full system access</p></div>';document.getElementById('stats').innerHTML=h}}catch(e){{alert('Invalid API key')}}}}
-<input type="text" id="crawl-input" placeholder="https://example.com" style="flex:1;padding:12px 16px;border-radius:12px;border:1px solid var(--border);background:var(--surface);color:var(--text);">
+async function buyCalls(){{document.getElementById('invoice-result').innerHTML='<p style="color:var(--gold)">Creating invoice...</p>';const r=await fetch('/api/create-invoice?api_key='+apiKey,{{method:'POST'}});const d=await r.json();if(d.checkout_url){{document.getElementById('invoice-result').innerHTML='<a href="'+d.checkout_url+'" target="_blank" class="btn btn-gold" style="width:100%">Pay '+d.amount+' with Bitcoin →</a><p style="color:var(--muted);margin-top:8px;font-size:0.85rem">Opens BTCPay checkout. Refresh after payment.</p>'}}else{{document.getElementById('invoice-result').innerHTML='<p style="color:var(--red)">Error: '+JSON.stringify(d)+'</p>'}}}}
-<button class="btn-primary" onclick="crawl()">Crawl</button>
+</script></body></html>"""
-</div>
+
-<div class="grid" id="stats"></div>
+
-<div id="output"></div>
+# ── Startup ──────────────────────────────────────────────────
-</div>
-<script>
-async function api(p){const r=await fetch(p);return r.json()}
-async function loadStatus(){
-try{const d=await api('/api/status');let h='';for(const[n,i]of Object.entries(d.services||{})){
-const c=i.status==='ok'||i.status==='green'?'ok':'down';
-let x='';if(i.documents!==undefined)x=i.documents+' docs';if(i.points!==undefined)x=i.points+' vectors';if(i.models!==undefined)x=i.models+' models';
-h+=`<div class="card"><div class="label">${n}</div><div class="value ${c}">${i.status||'down'}</div><div class="sub">${x||'—'}</div></div>`}
-document.getElementById('stats').innerHTML=h}catch(e){}
-}
-async function search(){
-const q=document.getElementById('query').value;if(!q)return;
-document.getElementById('output').innerHTML='<div class="loading">Searching...</div>';
-const d=await api('/api/search?q='+encodeURIComponent(q));
-let h=`<p style="color:var(--muted);margin-bottom:16px">${d.total||0} results for "${d.query||q}"</p><div class="results">`;
-for(const r of(d.hits||[]))h+=`<div class="result-card"><h3><a href="${r.url||'#'}" target="_blank">${r.title||'Untitled'}</a></h3><div class="excerpt">${r.excerpt||''}</div><div class="meta"><span>⭐ ${(r.score||0).toFixed(1)}</span>${r.category?`<span>📁 ${r.category}</span>`:''}<span>${r.crawled_at||''}</span></div></div>`;
-h+='</div>';document.getElementById('output').innerHTML=h}
-async function research(){
-const q=document.getElementById('query').value;if(!q)return;
-document.getElementById('output').innerHTML='<div class="loading">Deep researching... this may take 30-60 seconds</div>';
-const d=await api('/api/research?topic='+encodeURIComponent(q));
-let h=`<div class="card" style="margin-bottom:16px"><div class="label">Research Complete</div><div class="value" style="font-size:1.2rem">${d.topic}</div><div class="sub">Steps: ${(d.steps||[]).join(' → ')}</div></div>`;
-if(d.ai_summary)h+=`<div class="report"><h2>AI Summary</h2>${d.ai_summary.replace(/\\n/g,'<br>')}</div>`;
-h+=`<div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;margin-top:16px">`;
-h+=`<div class="card"><div class="label">Keyword Results</div><div class="sub">${d.keyword_results?.total||0} hits</div></div>`;
-h+=`<div class="card"><div class="label">Semantic Results</div><div class="sub">${d.semantic_results?.total||0} hits</div></div></div>`;
-document.getElementById('output').innerHTML=h}
-function toggleCrawl(){document.getElementById('crawl-box').classList.toggle('hidden')}
-async function crawl(){
-const u=document.getElementById('crawl-input').value;if(!u)return;
-document.getElementById('output').innerHTML='<div class="loading">Dispatching crawl...</div>';
-const d=await api('/api/crawl?url='+encodeURIComponent(u));
-document.getElementById('output').innerHTML='<pre>'+JSON.stringify(d,null,2)+'</pre>'}
-async function loadReport(){
-const q=document.getElementById('query').value||'AI research';
-document.getElementById('output').innerHTML='<div class="loading">Generating report...</div>';
-const d=await api('/api/report?topic='+encodeURIComponent(q));
-document.getElementById('output').innerHTML=`<div class="report"><h2>📊 Research Report: ${d.topic}</h2>${(d.report||'').replace(/\\n/g,'<br>')}<p style="color:var(--muted);margin-top:16px">Sources: ${d.sources_used||0}</p></div>`}
-loadStatus()
-</script></body></html>""")
 
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=8000)

server.py

@@ -1,110 +1,81 @@
 #!/usr/bin/env python3
-"""
+"""AI Research Engine — Thin MCP Proxy. Admin key for backend auth."""
-AI Research Engine — Thin MCP Proxy
-Runs on MacBook. Forwards all tool calls to the CT 145 backend.
-Minimal resource usage — all heavy lifting on Proxmox.
-"""
 
 import json
 import httpx
 from mcp.server import FastMCP
 
 BACKEND_URL = "http://10.30.20.249:8000"
+ADMIN_KEY = "sk-admin-unlimited-2026"
 client = httpx.Client(timeout=120.0)
 
-mcp = FastMCP(
-    "ai-research-engine",
-    instructions="""
-AI Research Engine — private knowledge acquisition system.
-
-search_web(query) — Full-text search across indexed documents
-semantic_search(query) — Find documents by meaning (vector search)
-crawl_url(url) — Crawl a URL into the index
-crawl_topic(topic) — Discover and crawl sources for a topic
-research_topic(topic) — Full pipeline: discover → crawl → summarize
-retrieve_document(url) — Get full content of an indexed document
-summarize_sources(urls, instruction) — AI summary of multiple sources
-extract_information(url, schema) — Structured data extraction
-create_report(topic, sources) — Generate comprehensive research report
-index_status() — System health and stats
-""",
-)
-
 def _get(path: str) -> dict:
-    r = client.get(f"{BACKEND_URL}{path}")
+    sep = "&" if "?" in path else "?"
+    r = client.get(f"{BACKEND_URL}{path}{sep}api_key={ADMIN_KEY}")
     r.raise_for_status()
     return r.json()
 
+mcp = FastMCP("ai-research-engine", instructions="AI Research Engine — private knowledge acquisition. 10 tools.")
 
 @mcp.tool()
 def search_web(query: str, category: str = "", limit: int = 10) -> str:
-    """Full-text search across indexed documents. Find by keywords, titles, content."""
+    """Full-text search across indexed documents."""
     r = _get(f"/api/search?q={query}&category={category}&limit={limit}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def semantic_search(query: str, limit: int = 10) -> str:
-    """Search by meaning using vector embeddings. Finds conceptually related docs."""
+    """Search by meaning using vector embeddings."""
     r = _get(f"/api/semantic-search?q={query}&limit={limit}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def crawl_url(url: str, depth: int = 1) -> str:
-    """Crawl a URL. depth: 0=just this page, 1=+linked pages."""
+    """Crawl a URL and index it."""
     r = _get(f"/api/crawl?url={url}&depth={depth}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def crawl_topic(topic: str, max_urls: int = 20) -> str:
-    """Discover and crawl sources for a topic using YaCy."""
+    """Discover and crawl sources for a topic."""
     r = _get(f"/api/crawl-topic?topic={topic}&max_urls={max_urls}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def research_topic(topic: str) -> str:
-    """Full research pipeline: keyword search → semantic search → crawl new sources → AI summary."""
+    """Full pipeline: search → crawl → AI summary."""
     r = _get(f"/api/research?topic={topic}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def retrieve_document(url: str) -> str:
-    """Get full indexed content of a document by URL."""
+    """Get full indexed content of a document."""
     r = _get(f"/api/document?url={url}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def summarize_sources(urls: str, instruction: str = "Summarize key points") -> str:
-    """Summarize multiple URLs using local LLM. urls: comma-separated."""
+    """AI summary of multiple URLs. urls: comma-separated."""
     r = _get(f"/api/summarize?urls={urls}&instruction={instruction}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
-def extract_information(url: str, schema: str = "company names, products, prices, specifications") -> str:
+def extract_information(url: str, schema: str = "company names, products, prices") -> str:
-    """Extract structured information from a document using LLM."""
+    """Extract structured data from a document using LLM."""
     r = _get(f"/api/extract?url={url}&schema={schema}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def create_report(topic: str, sources: str = "") -> str:
-    """Generate a comprehensive research report. sources: optional comma-separated URLs."""
+    """Generate comprehensive research report."""
     r = _get(f"/api/report?topic={topic}&sources={sources}")
     return json.dumps(r, indent=2)
 
-
 @mcp.tool()
 def index_status() -> str:
-    """Check health of all backend services: OpenSearch, Qdrant, YaCy, Ollama."""
+    """System health + business stats."""
     r = _get("/api/status")
     return json.dumps(r, indent=2)
 
-
 if __name__ == "__main__":
     mcp.run(transport="stdio")