Commercial launch: Postgres auth, BTCPay payments, premium UI
- Postgres: users, api_keys, usage_log, payments tables - Auth: API key system with rate limiting (402 when out of calls) - BTCPay: Bitcoin = 5 API calls, webhook for auto-credit - Admin: sk-admin-unlimited-2026 with unlimited calls - UI: Landing, Pricing (/5calls), Signup (no KYC), Dashboard with usage stats - Footer: Created by drjones + Buy Me a Coffee link - All endpoints auth-gated except signup, status, pages
This commit is contained in:
drjones
@@ -2,7 +2,7 @@ FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN pip install --no-cache-dir fastapi uvicorn httpx python-dotenv
|
||||
RUN pip install --no-cache-dir fastapi uvicorn httpx python-dotenv psycopg2-binary
|
||||
|
||||
COPY backend.py .
|
||||
|
||||
|
||||
890
backend.py
890
backend.py
@@ -1,547 +1,517 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AI Research Engine — Backend API
|
||||
Runs on CT 145, handles all heavy lifting.
|
||||
Exposes REST API consumed by the MCP proxy (MacBook) and dashboard.
|
||||
AI Research Engine — Commercial Backend
|
||||
Postgres auth, BTCPay payments, API key system, premium UI.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import os, json, hashlib, re, html as html_mod, random, secrets, time
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
import hashlib
|
||||
from fastapi import FastAPI, Query, HTTPException
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from fastapi import FastAPI, Query, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
|
||||
import uvicorn
|
||||
|
||||
app = FastAPI(title="AI Research Engine Backend")
|
||||
app = FastAPI(title="AI Research Engine")
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
||||
|
||||
# ── Config ──────────────────────────────────────────────────
|
||||
YACY_URL = os.getenv("YACY_URL", "http://localhost:8090")
|
||||
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200")
|
||||
YACY_URL = os.getenv("YACY_URL", "http://host.docker.internal:8090")
|
||||
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200")
|
||||
QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434")
|
||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "ornith:latest")
|
||||
INDEX_NAME = os.getenv("INDEX_NAME", "research_docs")
|
||||
BTCPAY_URL = "https://10.30.20.140"
|
||||
BTCPAY_KEY = "6026288e2e315984661c748baafd509e81a75f22"
|
||||
BTCPAY_STORE = "8ERS6v1UyQ46bQqaWr4LotvMCJLsmUhH2sT8zbzfjkT6"
|
||||
WEBHOOK_SECRET = "XjfhDd9DzXsUkA91B4SwRz"
|
||||
PRICE_USD = 5.00 # $5
|
||||
CALLS_PER_TIER = 5 # 5 API calls
|
||||
|
||||
# Proxies
|
||||
PROXIES = [
|
||||
"http://10.30.20.154:3128", # Tokyo
|
||||
"http://10.30.20.71:3128", # London
|
||||
"http://10.30.20.189:3128", # Sydney
|
||||
]
|
||||
|
||||
# DB
|
||||
def get_db():
|
||||
return psycopg2.connect(
|
||||
host="host.docker.internal", port=5432,
|
||||
user="research", password="ResearchDB2026!",
|
||||
database="research_engine"
|
||||
)
|
||||
|
||||
client = httpx.Client(timeout=30.0)
|
||||
ollama = httpx.Client(timeout=120.0, base_url=OLLAMA_URL)
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────
|
||||
# ── Auth Helpers ─────────────────────────────────────────────
|
||||
|
||||
def _auth(request: Request) -> dict:
|
||||
"""Authenticate by API key, return user row or raise 401."""
|
||||
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
||||
if not api_key:
|
||||
raise HTTPException(401, "API key required. Get one at /signup or /pricing.")
|
||||
db = get_db()
|
||||
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
||||
user = cur.fetchone()
|
||||
db.close()
|
||||
if not user:
|
||||
raise HTTPException(401, "Invalid API key.")
|
||||
if not user["is_admin"] and user["calls_remaining"] <= 0:
|
||||
raise HTTPException(402, "No calls remaining. Buy more at /pricing.")
|
||||
return dict(user)
|
||||
|
||||
def _track_usage(user_id: int, tool: str, request: Request):
|
||||
"""Deduct a call and log usage."""
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
cur.execute("UPDATE users SET calls_remaining = calls_remaining - 1, total_calls = total_calls + 1, last_used_at = NOW() WHERE id = %s AND NOT is_admin", (user_id,))
|
||||
cur.execute("INSERT INTO usage_log (user_id, tool_name, endpoint, ip_address) VALUES (%s,%s,%s,%s)",
|
||||
(user_id, tool, str(request.url), request.client.host if request.client else ""))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
# ── Search Helpers (unchanged from v1) ────────────────────────
|
||||
|
||||
def _ensure_index():
|
||||
try:
|
||||
client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}")
|
||||
except Exception:
|
||||
try:
|
||||
client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={
|
||||
"settings": {"number_of_shards": 1, "number_of_replicas": 0},
|
||||
"mappings": {"properties": {
|
||||
"url": {"type": "keyword"}, "title": {"type": "text"},
|
||||
"content": {"type": "text"}, "excerpt": {"type": "text"},
|
||||
"category": {"type": "keyword"}, "source_domain": {"type": "keyword"},
|
||||
"crawled_at": {"type": "date"}, "indexed_at": {"type": "date"},
|
||||
"metadata": {"type": "object"},
|
||||
}}
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _ensure_qdrant():
|
||||
try:
|
||||
client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}")
|
||||
except Exception:
|
||||
try:
|
||||
client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}", json={
|
||||
"vectors": {"size": 768, "distance": "Cosine"}
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
try: client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}")
|
||||
except: client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"url":{"type":"keyword"},"title":{"type":"text"},"content":{"type":"text"},"excerpt":{"type":"text"},"category":{"type":"keyword"},"source_domain":{"type":"keyword"},"crawled_at":{"type":"date"},"indexed_at":{"type":"date"},"metadata":{"type":"object"}}}})
|
||||
|
||||
def _ai_chat(prompt: str, system: str = "You are a research assistant. Be concise and factual.") -> str:
|
||||
r = ollama.post("/api/chat", json={
|
||||
"model": OLLAMA_MODEL, "messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt},
|
||||
], "stream": False,
|
||||
"options": {"temperature": 0.3, "num_predict": 2048},
|
||||
})
|
||||
r = ollama.post("/api/chat", json={"model":OLLAMA_MODEL,"messages":[{"role":"system","content":system},{"role":"user","content":prompt}],"stream":False,"options":{"temperature":0.3,"num_predict":2048}})
|
||||
body = r.json()
|
||||
return body.get("message", {}).get("content", "") or body.get("thinking", "") or ""
|
||||
return body.get("message",{}).get("content","") or body.get("thinking","") or ""
|
||||
|
||||
def _get_embedding(text: str) -> list:
|
||||
try:
|
||||
r = ollama.post("/api/embeddings", json={"model": "nomic-embed-text-v2-moe:latest", "prompt": text[:2048]})
|
||||
return r.json().get("embedding", [])
|
||||
except Exception:
|
||||
return []
|
||||
r = ollama.post("/api/embeddings", json={"model":"nomic-embed-text-v2-moe:latest","prompt":text[:2048]})
|
||||
return r.json().get("embedding",[])
|
||||
except: return []
|
||||
|
||||
def _fetch_and_index(url: str, category: str = ""):
|
||||
proxy_url = random.choice(PROXIES)
|
||||
try:
|
||||
pc = httpx.Client(proxy=proxy_url, timeout=15.0)
|
||||
r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"})
|
||||
if r.status_code != 200: return None
|
||||
html_text = r.text
|
||||
text = re.sub(r'<script[^>]*>.*?</script>','',html_text,flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<style[^>]*>.*?</style>','',text,flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<[^>]+>',' ',text)
|
||||
text = re.sub(r'\s+',' ',text).strip()
|
||||
text = html_mod.unescape(text)
|
||||
title_m = re.search(r'<title[^>]*>(.*?)</title>',html_text,re.IGNORECASE|re.DOTALL)
|
||||
title = html_mod.unescape(title_m.group(1).strip()) if title_m else url
|
||||
desc_m = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE)
|
||||
excerpt = desc_m.group(1)[:500] if desc_m else text[:500]
|
||||
domain = urlparse(url).netloc
|
||||
if not category:
|
||||
for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items():
|
||||
if k in domain: category = v; break
|
||||
if not category: category = "web"
|
||||
doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
|
||||
_ensure_index()
|
||||
client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"})
|
||||
emb = _get_embedding(excerpt[:1000])
|
||||
if emb:
|
||||
try:
|
||||
client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]})
|
||||
except: pass
|
||||
return {"title":title,"domain":domain,"category":category,"size":len(text)}
|
||||
except: return None
|
||||
|
||||
|
||||
# ── Status ────────────────────────────────────────────────────
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# PUBLIC PAGES
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def landing():
|
||||
return HTMLResponse(LANDING_HTML)
|
||||
|
||||
@app.get("/pricing", response_class=HTMLResponse)
|
||||
def pricing_page():
|
||||
return HTMLResponse(PRICING_HTML)
|
||||
|
||||
@app.get("/signup", response_class=HTMLResponse)
|
||||
def signup_page():
|
||||
return HTMLResponse(SIGNUP_HTML)
|
||||
|
||||
@app.get("/dashboard", response_class=HTMLResponse)
|
||||
def dashboard_page():
|
||||
return HTMLResponse(DASHBOARD_HTML)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# AUTH API
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@app.post("/api/signup")
|
||||
async def api_signup(request: Request):
|
||||
data = await request.json()
|
||||
email = data.get("email","").strip().lower()
|
||||
if not email or "@" not in email:
|
||||
return JSONResponse({"error":"Valid email required"}, 400)
|
||||
api_key = "sk-" + secrets.token_hex(24)
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
try:
|
||||
cur.execute("INSERT INTO users (email, password_hash, api_key, calls_remaining) VALUES (%s,%s,%s,0)", (email, "bcrypt_placeholder", api_key))
|
||||
db.commit()
|
||||
except psycopg2.errors.UniqueViolation:
|
||||
db.rollback()
|
||||
cur.execute("SELECT api_key FROM users WHERE email = %s", (email,))
|
||||
api_key = cur.fetchone()[0]
|
||||
db.close()
|
||||
return {"email": email, "api_key": api_key, "message": "Signup successful. Purchase API calls at /pricing to start using the engine."}
|
||||
|
||||
@app.get("/api/my-usage")
|
||||
async def my_usage(request: Request):
|
||||
# Allow even with 0 calls — users need to see their balance
|
||||
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
||||
if not api_key:
|
||||
raise HTTPException(401, "API key required.")
|
||||
db = get_db()
|
||||
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
||||
user = cur.fetchone()
|
||||
if not user:
|
||||
db.close()
|
||||
raise HTTPException(401, "Invalid API key.")
|
||||
user = dict(user)
|
||||
cur.execute("SELECT tool_name, COUNT(*) as cnt FROM usage_log WHERE user_id = %s GROUP BY tool_name ORDER BY cnt DESC", (user["id"],))
|
||||
user["usage_by_tool"] = [dict(r) for r in cur.fetchall()]
|
||||
db.close()
|
||||
return user
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# BTCPAY INTEGRATION
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@app.post("/api/create-invoice")
|
||||
async def create_invoice(request: Request):
|
||||
"""Create BTCPay invoice for $5 = 5 calls. Works even with 0 calls."""
|
||||
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
||||
if not api_key:
|
||||
raise HTTPException(401, "API key required.")
|
||||
db = get_db()
|
||||
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
||||
user = cur.fetchone()
|
||||
if not user:
|
||||
db.close()
|
||||
raise HTTPException(401, "Invalid API key.")
|
||||
user = dict(user)
|
||||
try:
|
||||
r = httpx.post(f"{BTCPAY_URL}/api/v1/stores/{BTCPAY_STORE}/invoices",
|
||||
headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"},
|
||||
json={
|
||||
"amount": str(PRICE_USD),
|
||||
"currency": "USD",
|
||||
"metadata": {
|
||||
"user_id": user["id"],
|
||||
"user_email": user["email"],
|
||||
"calls": CALLS_PER_TIER,
|
||||
"orderId": f"research-{user['id']}-{int(time.time())}"
|
||||
},
|
||||
"checkout": {"redirectURL": f"http://10.30.20.249:8000/dashboard"}
|
||||
},
|
||||
verify=False, timeout=15.0
|
||||
)
|
||||
inv = r.json()
|
||||
# Log pending payment
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
cur.execute("INSERT INTO payments (user_id, btcpay_invoice_id, amount_usd, calls_purchased, status) VALUES (%s,%s,%s,%s,'pending')",
|
||||
(user["id"], inv["id"], PRICE_USD, CALLS_PER_TIER))
|
||||
db.commit()
|
||||
db.close()
|
||||
return {"invoice_id": inv["id"], "checkout_url": inv["checkoutLink"], "amount": f"${PRICE_USD}", "calls": CALLS_PER_TIER}
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": str(e)}, 500)
|
||||
|
||||
@app.post("/webhook/btcpay")
|
||||
async def btcpay_webhook(request: Request):
|
||||
"""BTCPay calls this when invoice is settled."""
|
||||
body = await request.json()
|
||||
event_type = body.get("type", "")
|
||||
invoice_id = body.get("invoiceId", "")
|
||||
metadata = body.get("metadata", {})
|
||||
|
||||
# Handle nested metadata (BTCPay wraps it)
|
||||
if isinstance(metadata, dict):
|
||||
user_id = metadata.get("user_id")
|
||||
calls = metadata.get("calls", CALLS_PER_TIER)
|
||||
else:
|
||||
user_id = None
|
||||
calls = CALLS_PER_TIER
|
||||
|
||||
if event_type in ("InvoiceSettled", "InvoiceProcessing") and user_id:
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
cur.execute("UPDATE payments SET status = 'settled', settled_at = NOW() WHERE btcpay_invoice_id = %s", (invoice_id,))
|
||||
cur.execute("UPDATE users SET calls_remaining = calls_remaining + %s WHERE id = %s", (calls, user_id))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# API TOOLS (auth-gated)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@app.get("/api/status")
|
||||
def status():
|
||||
svc = {}
|
||||
try:
|
||||
try:
|
||||
r = client.get(f"{OPENSEARCH_URL}/_cluster/health")
|
||||
cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json() if r.status_code == 200 else {"count": 0}
|
||||
svc["opensearch"] = {"status": r.json().get("status", "down"), "documents": cnt.get("count", 0)}
|
||||
except Exception:
|
||||
svc["opensearch"] = {"status": "down"}
|
||||
cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json()
|
||||
svc["opensearch"] = {"status": r.json().get("status"), "documents": cnt.get("count",0)}
|
||||
except: svc["opensearch"] = {"status":"down"}
|
||||
try:
|
||||
r = client.get(f"{QDRANT_URL}/healthz")
|
||||
client.get(f"{QDRANT_URL}/healthz")
|
||||
col = client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}").json()
|
||||
svc["qdrant"] = {"status": "ok", "points": col.get("result", {}).get("points_count", 0)}
|
||||
except Exception:
|
||||
svc["qdrant"] = {"status": "down"}
|
||||
svc["qdrant"] = {"status":"ok","points":col.get("result",{}).get("points_count",0)}
|
||||
except: svc["qdrant"] = {"status":"down"}
|
||||
try:
|
||||
r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": "test", "maximumRecords": 1})
|
||||
svc["yacy"] = {"status": "ok" if r.status_code == 200 else "down"}
|
||||
except Exception:
|
||||
svc["yacy"] = {"status": "down"}
|
||||
client.get(f"{YACY_URL}/yacysearch.json",params={"query":"test","maximumRecords":1})
|
||||
svc["yacy"] = {"status":"ok"}
|
||||
except: svc["yacy"] = {"status":"down"}
|
||||
try:
|
||||
r = ollama.get("/api/tags")
|
||||
svc["ollama"] = {"status": "ok", "models": len(r.json().get("models", []))}
|
||||
except Exception:
|
||||
svc["ollama"] = {"status": "down"}
|
||||
return {"services": svc}
|
||||
|
||||
|
||||
# ── Search ────────────────────────────────────────────────────
|
||||
svc["ollama"] = {"status":"ok","models":len(r.json().get("models",[]))}
|
||||
except: svc["ollama"] = {"status":"down"}
|
||||
# DB stats
|
||||
try:
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM users")
|
||||
users = cur.fetchone()[0]
|
||||
cur.execute("SELECT COALESCE(SUM(total_calls),0) FROM users")
|
||||
total = cur.fetchone()[0]
|
||||
cur.execute("SELECT COUNT(*) FROM payments WHERE status = 'settled'")
|
||||
payments = cur.fetchone()[0]
|
||||
db.close()
|
||||
svc["business"] = {"status":"ok","users":users,"total_api_calls":total,"payments_settled":payments}
|
||||
except: svc["business"] = {"status":"down"}
|
||||
return {"services":svc}
|
||||
|
||||
@app.get("/api/search")
|
||||
def search_web(q: str = Query(...), category: str = "", limit: int = 10):
|
||||
def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "search_web", request)
|
||||
_ensure_index()
|
||||
body = {
|
||||
"size": limit,
|
||||
"query": {"bool": {"must": [{"multi_match": {"query": q, "fields": ["title^3", "content", "excerpt"]}}]}},
|
||||
"highlight": {"fields": {"content": {"fragment_size": 200, "number_of_fragments": 2}}},
|
||||
}
|
||||
if category:
|
||||
body["query"]["bool"]["filter"] = [{"term": {"category": category}}]
|
||||
body = {"size":limit,"query":{"bool":{"must":[{"multi_match":{"query":q,"fields":["title^3","content","excerpt"]}}]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":2}}}}
|
||||
if category: body["query"]["bool"]["filter"] = [{"term":{"category":category}}]
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body)
|
||||
result = r.json()
|
||||
hits = []
|
||||
for h in result.get("hits", {}).get("hits", []):
|
||||
for h in result.get("hits",{}).get("hits",[]):
|
||||
src = h["_source"]
|
||||
hits.append({"url": src.get("url"), "title": src.get("title"),
|
||||
"excerpt": src.get("excerpt") or (h.get("highlight", {}).get("content", [""])[0]),
|
||||
"category": src.get("category"), "crawled_at": src.get("crawled_at"), "score": h["_score"]})
|
||||
return {"query": q, "total": result.get("hits", {}).get("total", {}).get("value", 0), "hits": hits}
|
||||
hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]})
|
||||
return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits}
|
||||
except Exception as e:
|
||||
return {"query": q, "total": 0, "hits": [], "note": f"Index may be empty. {e}"}
|
||||
|
||||
return {"query":q,"total":0,"hits":[],"note":str(e)}
|
||||
|
||||
@app.get("/api/semantic-search")
|
||||
def semantic_search(q: str = Query(...), limit: int = 10):
|
||||
_ensure_qdrant()
|
||||
def semantic_search(request: Request, q: str = Query(...), limit: int = 10):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "semantic_search", request)
|
||||
emb = _get_embedding(q)
|
||||
if not emb:
|
||||
return {"hits": [], "error": "Embedding model not available"}
|
||||
if not emb: return {"hits":[],"error":"Embedding model unavailable"}
|
||||
try:
|
||||
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search", json={
|
||||
"vector": emb, "limit": limit, "with_payload": True, "with_vector": False})
|
||||
hits = [{"url": p.get("payload", {}).get("url"), "title": p.get("payload", {}).get("title"),
|
||||
"excerpt": str(p.get("payload", {}).get("excerpt", ""))[:300], "score": p.get("score")}
|
||||
for p in r.json().get("result", [])]
|
||||
return {"query": q, "total": len(hits), "hits": hits}
|
||||
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":limit,"with_payload":True,"with_vector":False})
|
||||
hits = [{"url":p.get("payload",{}).get("url"),"title":p.get("payload",{}).get("title"),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:300],"score":p.get("score")} for p in r.json().get("result",[])]
|
||||
return {"query":q,"total":len(hits),"hits":hits}
|
||||
except Exception as e:
|
||||
return {"hits": [], "error": str(e)}
|
||||
|
||||
|
||||
# ── Crawl ─────────────────────────────────────────────────────
|
||||
|
||||
import re
|
||||
import html as html_mod
|
||||
import random
|
||||
|
||||
# NordVPN HTTP proxies for geo-distributed fetching
|
||||
PROXIES = [
|
||||
"http://10.30.20.154:3128", # Tokyo, Japan
|
||||
"http://10.30.20.71:3128", # London, UK
|
||||
"http://10.30.20.189:3128", # Sydney, Australia
|
||||
]
|
||||
|
||||
def _fetch_and_index(url: str, category: str = ""):
|
||||
"""Fetch a URL, extract text, and index into OpenSearch immediately."""
|
||||
proxy_url = random.choice(PROXIES)
|
||||
try:
|
||||
# Create a per-request client with proxy
|
||||
proxy_client = httpx.Client(proxy=proxy_url, timeout=15.0)
|
||||
r = proxy_client.get(url,
|
||||
headers={"User-Agent": "Mozilla/5.0 (compatible; ResearchBot/1.0)"},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
html_text = r.text
|
||||
# Basic HTML-to-text
|
||||
text = re.sub(r'<script[^>]*>.*?</script>', '', html_text, flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL|re.IGNORECASE)
|
||||
text = re.sub(r'<[^>]+>', ' ', text)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
text = html_mod.unescape(text)
|
||||
# Extract title
|
||||
title_match = re.search(r'<title[^>]*>(.*?)</title>', html_text, re.IGNORECASE|re.DOTALL)
|
||||
title = html_mod.unescape(title_match.group(1).strip()) if title_match else url
|
||||
# Extract meta description
|
||||
desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)', html_text, re.IGNORECASE)
|
||||
excerpt = desc_match.group(1)[:500] if desc_match else text[:500]
|
||||
# Derive domain + category
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(url).netloc
|
||||
if not category:
|
||||
cat_map = {"wikipedia": "reference", "github": "software", "arxiv": "science",
|
||||
"docs.": "documentation", "blog.": "blog", "news.": "news"}
|
||||
for k, v in cat_map.items():
|
||||
if k in domain:
|
||||
category = v
|
||||
break
|
||||
if not category:
|
||||
category = "web"
|
||||
# Index into OpenSearch
|
||||
import datetime
|
||||
doc = {
|
||||
"url": url, "title": title, "content": text[:50000],
|
||||
"excerpt": excerpt[:1000], "category": category,
|
||||
"source_domain": domain,
|
||||
"crawled_at": datetime.datetime.utcnow().isoformat(),
|
||||
"indexed_at": datetime.datetime.utcnow().isoformat(),
|
||||
}
|
||||
_ensure_index()
|
||||
client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",
|
||||
json=doc, params={"refresh": "true"})
|
||||
# Also index into Qdrant
|
||||
emb = _get_embedding(excerpt[:1000])
|
||||
if emb:
|
||||
try:
|
||||
_ensure_qdrant()
|
||||
client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points", json={
|
||||
"points": [{"id": hashlib.md5(url.encode()).hexdigest(),
|
||||
"vector": emb, "payload": {"url": url, "title": title, "excerpt": excerpt[:500]}}]
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return {"title": title, "domain": domain, "category": category, "size": len(text)}
|
||||
except Exception as e:
|
||||
return None
|
||||
|
||||
return {"hits":[],"error":str(e)}
|
||||
|
||||
@app.get("/api/crawl")
|
||||
def crawl_url(url: str = Query(...), depth: int = 1):
|
||||
# 1. Fetch and index immediately into OpenSearch + Qdrant
|
||||
def crawl_url(request: Request, url: str = Query(...), depth: int = 1):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "crawl_url", request)
|
||||
indexed = _fetch_and_index(url)
|
||||
# 2. Also submit to YaCy for deeper crawling
|
||||
try:
|
||||
r = client.get(f"{YACY_URL}/Crawler_p.json", params={
|
||||
"crawlingDomMaxPages": 50, "crawlingDepth": depth,
|
||||
"crawlingStart": url, "crawlingQ": "on",
|
||||
"bookmarkTitle": "research", "bookmarkFolder": "/research",
|
||||
"indexText": "on", "indexMedia": "on",
|
||||
"crawlingMode": "url", "cachePolicy": "iffresh",
|
||||
})
|
||||
return {"status": "crawl_started", "url": url, "depth": depth, "indexed": indexed}
|
||||
except Exception as e:
|
||||
return {"status": "indexed_only", "url": url, "indexed": indexed, "yacy_error": str(e)}
|
||||
client.get(f"{YACY_URL}/Crawler_p.json", params={"crawlingDomMaxPages":50,"crawlingDepth":depth,"crawlingStart":url,"crawlingQ":"on","bookmarkTitle":"research","bookmarkFolder":"/research","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"})
|
||||
return {"status":"crawl_started","url":url,"depth":depth,"indexed":indexed}
|
||||
except:
|
||||
return {"status":"indexed_only","url":url,"indexed":indexed}
|
||||
|
||||
|
||||
@app.get("/api/crawl-topic")
|
||||
def crawl_topic(topic: str = Query(...), max_urls: int = 20):
|
||||
discovered = []
|
||||
@app.get("/api/research")
|
||||
def research_topic(request: Request, topic: str = Query(...)):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "research_topic", request)
|
||||
steps = []; kw_result = {"hits":[]}; sem_result = {"hits":[]}
|
||||
try:
|
||||
r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": topic, "maximumRecords": max_urls, "resource": "global"})
|
||||
for ch in r.json().get("channels", []):
|
||||
for item in ch.get("items", []):
|
||||
if item.get("link"):
|
||||
discovered.append({"url": item["link"], "title": item.get("title", ""), "snippet": item.get("description", "")})
|
||||
except Exception as e:
|
||||
return {"topic": topic, "error": f"Discovery failed: {e}", "urls_discovered": 0, "urls_crawled": 0}
|
||||
|
||||
crawled = 0
|
||||
for u in discovered[:max_urls]:
|
||||
try:
|
||||
client.get(f"{YACY_URL}/Crawler_p.json", params={
|
||||
"crawlingDomMaxPages": 10, "crawlingDepth": 0,
|
||||
"crawlingStart": u["url"], "crawlingQ": "on",
|
||||
"indexText": "on", "indexMedia": "on",
|
||||
"crawlingMode": "url", "cachePolicy": "iffresh",
|
||||
}, timeout=5.0)
|
||||
crawled += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"topic": topic, "urls_discovered": len(discovered), "urls_crawled": crawled}
|
||||
|
||||
|
||||
# ── Document Retrieval ────────────────────────────────────────
|
||||
|
||||
@app.get("/api/document")
|
||||
def retrieve_document(url: str = Query(...)):
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":5,"query":{"multi_match":{"query":topic,"fields":["title^3","content","excerpt"]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":1}}}})
|
||||
hits = [{"title":h["_source"].get("title"),"excerpt":h.get("highlight",{}).get("content",[h["_source"].get("excerpt","")])[0]} for h in r.json().get("hits",{}).get("hits",[])]
|
||||
kw_result = {"total":len(hits),"hits":hits}; steps.append("keyword_search_done")
|
||||
except: steps.append("keyword_search_skipped")
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
|
||||
"size": 1, "query": {"term": {"url": url}}})
|
||||
hits = r.json().get("hits", {}).get("hits", [])
|
||||
if not hits:
|
||||
return {"error": "Not found", "url": url}
|
||||
src = hits[0]["_source"]
|
||||
return {"url": src.get("url"), "title": src.get("title"),
|
||||
"content": src.get("content", "")[:10000], "excerpt": src.get("excerpt"),
|
||||
"category": src.get("category"), "crawled_at": src.get("crawled_at")}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "url": url}
|
||||
|
||||
|
||||
# ── AI Synthesis ──────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/summarize")
|
||||
def summarize_sources(urls: str = Query(...), instruction: str = "Summarize key points"):
|
||||
url_list = [u.strip() for u in urls.split(",") if u.strip()]
|
||||
combined = ""
|
||||
for url in url_list[:5]:
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
|
||||
hits = r.json().get("hits", {}).get("hits", [])
|
||||
if hits:
|
||||
src = hits[0]["_source"]
|
||||
combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content', src.get('excerpt',''))[:2000]}"
|
||||
except Exception:
|
||||
pass
|
||||
if not combined.strip():
|
||||
return {"error": "No content retrieved"}
|
||||
summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.")
|
||||
return {"instruction": instruction, "sources": len(url_list), "summary": summary}
|
||||
|
||||
|
||||
@app.get("/api/extract")
|
||||
def extract_information(url: str = Query(...), schema: str = Query("company names, products, prices")):
|
||||
emb = _get_embedding(topic)
|
||||
if emb:
|
||||
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":5,"with_payload":True})
|
||||
hits2 = [{"title":p.get("payload",{}).get("title",""),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:200],"score":p.get("score")} for p in r.json().get("result",[])]
|
||||
sem_result = {"total":len(hits2),"hits":hits2}; steps.append("semantic_search_done")
|
||||
except: steps.append("semantic_search_skipped")
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
|
||||
hits = r.json().get("hits", {}).get("hits", [])
|
||||
if not hits:
|
||||
return {"error": "Not found", "url": url}
|
||||
content = hits[0]["_source"].get("content", "")[:8000]
|
||||
prompt = f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON."
|
||||
result = _ai_chat(prompt, system="Extract structured data. Return ONLY valid JSON. No explanation.")
|
||||
return {"url": url, "schema": schema, "extracted": result}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
r = client.get(f"{YACY_URL}/yacysearch.json",params={"query":topic,"maximumRecords":10,"resource":"global"})
|
||||
discovered = [item.get("link") for ch in r.json().get("channels",[]) for item in ch.get("items",[]) if item.get("link")]
|
||||
for url in discovered[:10]:
|
||||
try: client.get(f"{YACY_URL}/Crawler_p.json",params={"crawlingDomMaxPages":10,"crawlingDepth":0,"crawlingStart":url,"crawlingQ":"on","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"},timeout=5.0)
|
||||
except: pass
|
||||
steps.append(f"crawl_dispatched_{len(discovered[:10])}")
|
||||
except: steps.append("crawl_skipped")
|
||||
all_src = ""
|
||||
for h in kw_result.get("hits",[])[:3] + sem_result.get("hits",[])[:3]:
|
||||
all_src += f"- {h.get('title','Unknown')}: {h.get('excerpt','')[:200]}\n"
|
||||
summary = _ai_chat(f"Research topic: {topic}\n\nSources:\n{all_src}\n\nConcise research summary (3-5 paragraphs): key findings, important sources, knowledge gaps, next steps.") if all_src else ""
|
||||
return {"topic":topic,"steps":steps,"keyword_results":kw_result,"semantic_results":sem_result,"ai_summary":summary}
|
||||
|
||||
@app.get("/api/report")
|
||||
def create_report(topic: str = Query(...), sources: str = ""):
|
||||
def create_report(request: Request, topic: str = Query(...), sources: str = ""):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "create_report", request)
|
||||
if sources:
|
||||
urls = [u.strip() for u in sources.split(",") if u.strip()]
|
||||
else:
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
|
||||
"size": 8, "query": {"multi_match": {"query": topic, "fields": ["title^3", "content"]}}})
|
||||
urls = [h["_source"]["url"] for h in r.json().get("hits", {}).get("hits", [])]
|
||||
except Exception:
|
||||
urls = []
|
||||
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":8,"query":{"multi_match":{"query":topic,"fields":["title^3","content"]}}})
|
||||
urls = [h["_source"]["url"] for h in r.json().get("hits",{}).get("hits",[])]
|
||||
except: urls = []
|
||||
gathered = ""
|
||||
for url in urls[:8]:
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size": 1, "query": {"term": {"url": url}}})
|
||||
hits = r.json().get("hits", {}).get("hits", [])
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
||||
hits = r.json().get("hits",{}).get("hits",[])
|
||||
if hits:
|
||||
src = hits[0]["_source"]
|
||||
gathered += f"\n\n### {src.get('title','Source')}\nURL: {url}\n{src.get('content',src.get('excerpt',''))[:1500]}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
prompt = f"""Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}
|
||||
|
||||
Generate a comprehensive report:
|
||||
1. Executive Summary
|
||||
2. Key Findings (numbered)
|
||||
3. Source Analysis
|
||||
4. Knowledge Gaps
|
||||
5. Recommendations
|
||||
|
||||
Be thorough, use markdown, cite sources."""
|
||||
report = _ai_chat(prompt, system="You are a senior research analyst. Produce thorough, structured reports.")
|
||||
return {"topic": topic, "sources_used": len(urls), "report": report}
|
||||
|
||||
|
||||
# ── Research Pipeline ─────────────────────────────────────────
|
||||
|
||||
@app.get("/api/research")
|
||||
def research_topic(topic: str = Query(...)):
|
||||
steps = []
|
||||
kw_result = {"hits": []}
|
||||
sem_result = {"hits": []}
|
||||
except: pass
|
||||
report = _ai_chat(f"Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}\n\nGenerate a comprehensive report:\n1. Executive Summary\n2. Key Findings (numbered)\n3. Source Analysis\n4. Knowledge Gaps\n5. Recommendations\n\nBe thorough, use markdown, cite sources.", system="You are a senior research analyst. Produce thorough, structured reports.")
|
||||
return {"topic":topic,"sources_used":len(urls),"report":report}
|
||||
|
||||
@app.get("/api/document")
|
||||
def retrieve_document(request: Request, url: str = Query(...)):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "retrieve_document", request)
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
|
||||
"size": 5, "query": {"multi_match": {"query": topic, "fields": ["title^3", "content", "excerpt"]}},
|
||||
"highlight": {"fields": {"content": {"fragment_size": 200, "number_of_fragments": 1}}},
|
||||
})
|
||||
hits = [{"title": h["_source"].get("title"), "excerpt": h.get("highlight", {}).get("content", [h["_source"].get("excerpt", "")])[0]}
|
||||
for h in r.json().get("hits", {}).get("hits", [])]
|
||||
kw_result = {"total": len(hits), "hits": hits}
|
||||
steps.append("keyword_search_done")
|
||||
except Exception:
|
||||
steps.append("keyword_search_skipped")
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
||||
hits = r.json().get("hits",{}).get("hits",[])
|
||||
if not hits: return {"error":"Not found","url":url}
|
||||
src = hits[0]["_source"]
|
||||
return {"url":src.get("url"),"title":src.get("title"),"content":src.get("content","")[:10000],"excerpt":src.get("excerpt"),"category":src.get("category"),"crawled_at":src.get("crawled_at")}
|
||||
except Exception as e:
|
||||
return {"error":str(e),"url":url}
|
||||
|
||||
@app.get("/api/summarize")
|
||||
def summarize_sources(request: Request, urls: str = Query(...), instruction: str = "Summarize key points"):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "summarize_sources", request)
|
||||
url_list = [u.strip() for u in urls.split(",") if u.strip()]
|
||||
combined = ""
|
||||
for url in url_list[:5]:
|
||||
try:
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
||||
hits = r.json().get("hits",{}).get("hits",[])
|
||||
if hits:
|
||||
src = hits[0]["_source"]
|
||||
combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content',src.get('excerpt',''))[:2000]}"
|
||||
except: pass
|
||||
if not combined.strip(): return {"error":"No content retrieved"}
|
||||
summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.")
|
||||
return {"instruction":instruction,"sources":len(url_list),"summary":summary}
|
||||
|
||||
@app.get("/api/extract")
|
||||
def extract_information(request: Request, url: str = Query(...), schema: str = Query("company names, products, prices")):
|
||||
user = _auth(request)
|
||||
_track_usage(user["id"], "extract_information", request)
|
||||
try:
|
||||
emb = _get_embedding(topic)
|
||||
if emb:
|
||||
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search", json={
|
||||
"vector": emb, "limit": 5, "with_payload": True})
|
||||
hits2 = [{"title": p.get("payload", {}).get("title", ""), "excerpt": str(p.get("payload", {}).get("excerpt", ""))[:200],
|
||||
"score": p.get("score")} for p in r.json().get("result", [])]
|
||||
sem_result = {"total": len(hits2), "hits": hits2}
|
||||
steps.append("semantic_search_done")
|
||||
except Exception:
|
||||
steps.append("semantic_search_skipped")
|
||||
|
||||
try:
|
||||
r = client.get(f"{YACY_URL}/yacysearch.json", params={"query": topic, "maximumRecords": 10, "resource": "global"})
|
||||
discovered = [item.get("link") for ch in r.json().get("channels", []) for item in ch.get("items", []) if item.get("link")]
|
||||
for url in discovered[:10]:
|
||||
try:
|
||||
client.get(f"{YACY_URL}/Crawler_p.json", params={
|
||||
"crawlingDomMaxPages": 10, "crawlingDepth": 0, "crawlingStart": url,
|
||||
"crawlingQ": "on", "indexText": "on", "indexMedia": "on",
|
||||
"crawlingMode": "url", "cachePolicy": "iffresh"}, timeout=5.0)
|
||||
except Exception:
|
||||
pass
|
||||
steps.append(f"crawl_dispatched_{len(discovered[:10])}")
|
||||
except Exception:
|
||||
steps.append("crawl_skipped")
|
||||
|
||||
all_sources = ""
|
||||
for h in kw_result.get("hits", [])[:3] + sem_result.get("hits", [])[:3]:
|
||||
all_sources += f"- {h.get('title', 'Unknown')}: {h.get('excerpt', '')[:200]}\n"
|
||||
|
||||
summary = ""
|
||||
if all_sources:
|
||||
summary = _ai_chat(
|
||||
f"Research topic: {topic}\n\nSources:\n{all_sources}\n\nConcise research summary (3-5 paragraphs): "
|
||||
"key findings, important sources, knowledge gaps, next steps.")
|
||||
|
||||
return {"topic": topic, "steps": steps,
|
||||
"keyword_results": kw_result, "semantic_results": sem_result,
|
||||
"ai_summary": summary}
|
||||
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
||||
hits = r.json().get("hits",{}).get("hits",[])
|
||||
if not hits: return {"error":"Not found","url":url}
|
||||
content = hits[0]["_source"].get("content","")[:8000]
|
||||
result = _ai_chat(f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON.", system="Extract structured data. Return ONLY valid JSON.")
|
||||
return {"url":url,"schema":schema,"extracted":result}
|
||||
except Exception as e:
|
||||
return {"error":str(e)}
|
||||
|
||||
|
||||
# ── Dashboard ─────────────────────────────────────────────────
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# UI TEMPLATES
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def dashboard():
|
||||
return HTMLResponse("""
|
||||
<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
:root{--bg:#09090b;--surface:#18181b;--border:#27272a;--accent:#7c3aed;--accent2:#a855f7;--text:#f4f4f5;--muted:#a1a1aa;--green:#22c55e;--red:#ef4444;--amber:#f59e0b}
|
||||
CSS = """
|
||||
:root{--bg:#050508;--surface:#0c0c14;--border:#1e1e30;--accent:#7c3aed;--accent2:#a855f7;--gold:#f59e0b;--text:#f4f4f5;--muted:#a1a1aa;--green:#22c55e;--red:#ef4444}
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh}
|
||||
.header{background:var(--surface);border-bottom:1px solid var(--border);padding:20px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:10}
|
||||
.header h1{font-size:1.4rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--accent2),#ec4899);-webkit-background-clip:text;-webkit-text-fill-color:transparent;letter-spacing:-0.02em}
|
||||
.header .badge{font-size:0.75rem;color:var(--muted);background:var(--border);padding:6px 12px;border-radius:999px}
|
||||
.container{max-width:1400px;margin:0 auto;padding:32px 24px}
|
||||
.hero{text-align:center;padding:48px 0 32px}
|
||||
.hero h2{font-size:2.2rem;font-weight:800;letter-spacing:-0.03em;margin-bottom:12px}
|
||||
.hero p{color:var(--muted);font-size:1.1rem;max-width:600px;margin:0 auto}
|
||||
.search-box{display:flex;gap:12px;max-width:800px;margin:0 auto 40px}
|
||||
.search-box input{flex:1;padding:16px 24px;border-radius:16px;border:1px solid var(--border);background:var(--surface);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s}
|
||||
.search-box input:focus{border-color:var(--accent)}
|
||||
.search-box button{padding:16px 32px;border-radius:16px;border:none;font-weight:600;font-size:1rem;cursor:pointer;transition:all .2s}
|
||||
body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh;overflow-x:hidden}
|
||||
.gradient-bg{background:radial-gradient(ellipse 80% 50% at 50% -20%,rgba(120,60,255,0.15),transparent)}
|
||||
.header{background:rgba(12,12,20,0.8);backdrop-filter:blur(20px);border-bottom:1px solid var(--border);padding:16px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:100}
|
||||
.header h1{font-size:1.3rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--accent2),#ec4899);-webkit-background-clip:text;-webkit-text-fill-color:transparent}
|
||||
.header nav{display:flex;gap:20px;align-items:center}
|
||||
.header nav a{color:var(--muted);text-decoration:none;font-size:0.9rem;transition:color .2s}
|
||||
.header nav a:hover{color:var(--text)}
|
||||
.btn{padding:12px 24px;border-radius:12px;border:none;font-weight:600;font-size:0.95rem;cursor:pointer;transition:all .2s;text-decoration:none;display:inline-block}
|
||||
.btn-primary{background:var(--accent);color:white}
|
||||
.btn-primary:hover{background:var(--accent2)}
|
||||
.btn-secondary{background:var(--surface);color:var(--text);border:1px solid var(--border)}
|
||||
.btn-secondary:hover{border-color:var(--accent)}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:16px;margin-bottom:32px}
|
||||
.card{background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:24px;transition:border-color .2s}
|
||||
.card:hover{border-color:var(--accent)}
|
||||
.card .label{color:var(--muted);font-size:0.8rem;text-transform:uppercase;letter-spacing:0.05em;margin-bottom:8px}
|
||||
.card .value{font-size:1.8rem;font-weight:700}
|
||||
.card .sub{color:var(--muted);font-size:0.85rem;margin-top:4px}
|
||||
.ok{color:var(--green)}.down{color:var(--red)}.warn{color:var(--amber)}
|
||||
.action-bar{display:flex;gap:10px;margin-bottom:28px;flex-wrap:wrap}
|
||||
.action-bar button{padding:10px 20px;border-radius:10px;border:1px solid var(--border);background:var(--surface);color:var(--text);cursor:pointer;font-size:0.9rem;transition:all .15s}
|
||||
.action-bar button:hover{border-color:var(--accent);background:#1f1f23}
|
||||
.results{display:flex;flex-direction:column;gap:12px}
|
||||
.result-card{background:var(--surface);border:1px solid var(--border);border-radius:14px;padding:20px;transition:border-color .15s}
|
||||
.result-card:hover{border-color:var(--accent)}
|
||||
.result-card h3{margin-bottom:6px;font-size:1.1rem}
|
||||
.result-card h3 a{color:var(--accent2);text-decoration:none}
|
||||
.result-card h3 a:hover{text-decoration:underline}
|
||||
.result-card .excerpt{color:var(--muted);font-size:0.9rem;line-height:1.5}
|
||||
.result-card .meta{color:#71717a;font-size:0.78rem;margin-top:10px;display:flex;gap:16px}
|
||||
.loading{text-align:center;padding:48px;color:var(--muted)}
|
||||
.hidden{display:none}
|
||||
pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:20px;overflow-x:auto;font-size:0.85rem;line-height:1.6;white-space:pre-wrap}
|
||||
.report{background:var(--surface);border:1px solid var(--border);border-radius:14px;padding:24px;line-height:1.7}
|
||||
.report h1,.report h2,.report h3{color:var(--accent2);margin:16px 0 8px}
|
||||
.report ul,.report ol{padding-left:20px;margin:8px 0}
|
||||
.report li{margin:4px 0}
|
||||
</style></head><body>
|
||||
<div class="header"><h1>🧠 AI Research Engine</h1><div class="badge">Self-hosted · Private</div></div>
|
||||
<div class="container">
|
||||
<div class="hero"><h2>Your Private Research Cloud</h2><p>Discover, index, and synthesize knowledge — all on your own infrastructure.</p></div>
|
||||
<div class="search-box">
|
||||
<input type="text" id="query" placeholder="Research anything..." onkeydown="if(event.key==='Enter')search()">
|
||||
<button class="btn-primary" onclick="search()">🔍 Search</button>
|
||||
<button class="btn-secondary" onclick="research()" style="background:var(--accent);color:white;">🧪 Deep Research</button>
|
||||
</div>
|
||||
<div class="action-bar">
|
||||
<button onclick="loadStatus()">📡 System Status</button>
|
||||
<button onclick="toggleCrawl()">🕷️ Crawl URL</button>
|
||||
<button onclick="loadReport()">📄 Generate Report</button>
|
||||
</div>
|
||||
<div id="crawl-box" class="hidden" style="margin-bottom:20px;display:flex;gap:10px;">
|
||||
<input type="text" id="crawl-input" placeholder="https://example.com" style="flex:1;padding:12px 16px;border-radius:12px;border:1px solid var(--border);background:var(--surface);color:var(--text);">
|
||||
<button class="btn-primary" onclick="crawl()">Crawl</button>
|
||||
</div>
|
||||
<div class="grid" id="stats"></div>
|
||||
<div id="output"></div>
|
||||
</div>
|
||||
<script>
|
||||
async function api(p){const r=await fetch(p);return r.json()}
|
||||
async function loadStatus(){
|
||||
try{const d=await api('/api/status');let h='';for(const[n,i]of Object.entries(d.services||{})){
|
||||
const c=i.status==='ok'||i.status==='green'?'ok':'down';
|
||||
let x='';if(i.documents!==undefined)x=i.documents+' docs';if(i.points!==undefined)x=i.points+' vectors';if(i.models!==undefined)x=i.models+' models';
|
||||
h+=`<div class="card"><div class="label">${n}</div><div class="value ${c}">${i.status||'down'}</div><div class="sub">${x||'—'}</div></div>`}
|
||||
document.getElementById('stats').innerHTML=h}catch(e){}
|
||||
}
|
||||
async function search(){
|
||||
const q=document.getElementById('query').value;if(!q)return;
|
||||
document.getElementById('output').innerHTML='<div class="loading">Searching...</div>';
|
||||
const d=await api('/api/search?q='+encodeURIComponent(q));
|
||||
let h=`<p style="color:var(--muted);margin-bottom:16px">${d.total||0} results for "${d.query||q}"</p><div class="results">`;
|
||||
for(const r of(d.hits||[]))h+=`<div class="result-card"><h3><a href="${r.url||'#'}" target="_blank">${r.title||'Untitled'}</a></h3><div class="excerpt">${r.excerpt||''}</div><div class="meta"><span>⭐ ${(r.score||0).toFixed(1)}</span>${r.category?`<span>📁 ${r.category}</span>`:''}<span>${r.crawled_at||''}</span></div></div>`;
|
||||
h+='</div>';document.getElementById('output').innerHTML=h}
|
||||
async function research(){
|
||||
const q=document.getElementById('query').value;if(!q)return;
|
||||
document.getElementById('output').innerHTML='<div class="loading">Deep researching... this may take 30-60 seconds</div>';
|
||||
const d=await api('/api/research?topic='+encodeURIComponent(q));
|
||||
let h=`<div class="card" style="margin-bottom:16px"><div class="label">Research Complete</div><div class="value" style="font-size:1.2rem">${d.topic}</div><div class="sub">Steps: ${(d.steps||[]).join(' → ')}</div></div>`;
|
||||
if(d.ai_summary)h+=`<div class="report"><h2>AI Summary</h2>${d.ai_summary.replace(/\\n/g,'<br>')}</div>`;
|
||||
h+=`<div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;margin-top:16px">`;
|
||||
h+=`<div class="card"><div class="label">Keyword Results</div><div class="sub">${d.keyword_results?.total||0} hits</div></div>`;
|
||||
h+=`<div class="card"><div class="label">Semantic Results</div><div class="sub">${d.semantic_results?.total||0} hits</div></div></div>`;
|
||||
document.getElementById('output').innerHTML=h}
|
||||
function toggleCrawl(){document.getElementById('crawl-box').classList.toggle('hidden')}
|
||||
async function crawl(){
|
||||
const u=document.getElementById('crawl-input').value;if(!u)return;
|
||||
document.getElementById('output').innerHTML='<div class="loading">Dispatching crawl...</div>';
|
||||
const d=await api('/api/crawl?url='+encodeURIComponent(u));
|
||||
document.getElementById('output').innerHTML='<pre>'+JSON.stringify(d,null,2)+'</pre>'}
|
||||
async function loadReport(){
|
||||
const q=document.getElementById('query').value||'AI research';
|
||||
document.getElementById('output').innerHTML='<div class="loading">Generating report...</div>';
|
||||
const d=await api('/api/report?topic='+encodeURIComponent(q));
|
||||
document.getElementById('output').innerHTML=`<div class="report"><h2>📊 Research Report: ${d.topic}</h2>${(d.report||'').replace(/\\n/g,'<br>')}<p style="color:var(--muted);margin-top:16px">Sources: ${d.sources_used||0}</p></div>`}
|
||||
loadStatus()
|
||||
</script></body></html>""")
|
||||
.btn-primary:hover{background:var(--accent2);transform:translateY(-1px);box-shadow:0 8px 25px rgba(124,58,237,0.3)}
|
||||
.btn-gold{background:linear-gradient(135deg,var(--gold),#fbbf24);color:#1a1a1a}
|
||||
.btn-gold:hover{transform:translateY(-1px);box-shadow:0 8px 25px rgba(245,158,11,0.3)}
|
||||
.btn-outline{background:transparent;border:1px solid var(--border);color:var(--text)}
|
||||
.container{max-width:1200px;margin:0 auto;padding:40px 24px}
|
||||
.hero{text-align:center;padding:80px 0 60px}
|
||||
.hero h2{font-size:3rem;font-weight:900;letter-spacing:-0.04em;line-height:1.1;margin-bottom:20px;background:linear-gradient(135deg,var(--text),var(--accent2));-webkit-background-clip:text;-webkit-text-fill-color:transparent}
|
||||
.hero p{color:var(--muted);font-size:1.2rem;max-width:650px;margin:0 auto 32px;line-height:1.6}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:20px;margin:40px 0}
|
||||
.card{background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:28px;transition:all .2s}
|
||||
.card:hover{border-color:var(--accent);transform:translateY(-2px)}
|
||||
.card h3{font-size:1.2rem;margin-bottom:10px}
|
||||
.card p{color:var(--muted);font-size:0.9rem;line-height:1.5}
|
||||
.price{font-size:3rem;font-weight:900;text-align:center;margin:20px 0}
|
||||
.price span{font-size:1rem;color:var(--muted);font-weight:400}
|
||||
.feature-list{list-style:none;margin:20px 0}
|
||||
.feature-list li{padding:8px 0;color:var(--muted);font-size:0.9rem}
|
||||
.feature-list li::before{content:'✓ ';color:var(--green);font-weight:700}
|
||||
pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:16px;overflow-x:auto;font-size:0.85rem;margin:10px 0}
|
||||
.footer{text-align:center;padding:40px;color:var(--muted);font-size:0.85rem;border-top:1px solid var(--border);margin-top:60px}
|
||||
.footer a{color:var(--accent2);text-decoration:none}
|
||||
input,textarea{width:100%;padding:14px 18px;border-radius:12px;border:1px solid var(--border);background:var(--bg);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s;margin-bottom:12px}
|
||||
input:focus,textarea:focus{border-color:var(--accent)}
|
||||
"""
|
||||
|
||||
LANDING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>AI Research Engine — Private Knowledge Cloud</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}.glow{{position:absolute;width:600px;height:600px;border-radius:50%;filter:blur(120px);opacity:0.12;pointer-events:none}}.glow-1{{background:var(--accent);top:-200px;left:-100px}}.glow-2{{background:#ec4899;bottom:-200px;right:-100px}}</style></head><body class="gradient-bg"><div class="glow glow-1"></div><div class="glow glow-2"></div><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>The Ultimate Source of<br>Uncensored Internet Knowledge</h2><p>A self-hosted AI workforce that continuously gathers, organizes, and acts on information. Your private research cloud — no filters, no tracking, no limits.</p><div style="display:flex;gap:16px;justify-content:center;flex-wrap:wrap"><a href="/signup" class="btn btn-primary" style="font-size:1.1rem;padding:16px 36px">Get Started — Free Signup</a><a href="/pricing" class="btn btn-gold" style="font-size:1.1rem;padding:16px 36px">$5 Bitcoin = 5 API Calls ⚡</a></div></div><div class="grid"><div class="card"><h3>🔍 Deep Search</h3><p>Full-text search across millions of indexed documents. Find anything — no Google bubble, no censorship.</p></div><div class="card"><h3>🧬 Semantic Understanding</h3><p>Search by meaning, not keywords. Our vector engine finds conceptually related content even when the words differ.</p></div><div class="card"><h3>🕷️ Autonomous Crawling</h3><p>Point it at a topic and it discovers, crawls, and indexes every relevant source. Never miss a thing.</p></div><div class="card"><h3>🤖 AI Synthesis</h3><p>Local LLMs summarize, extract, and report. Your research, analyzed by AI running on our hardware.</p></div><div class="card"><h3>🌍 Geo-Distributed</h3><p>Crawls route through proxies in Tokyo, London, and Sydney. Bypass regional blocks automatically.</p></div><div class="card"><h3>🔐 Private & Self-Hosted</h3><p>Everything runs on our metal. No third-party APIs, no data harvesting, no surveillance.</p></div></div><div style="text-align:center;padding:40px 0"><h3 style="font-size:1.5rem;margin-bottom:16px">Trusted by AI Agents Worldwide</h3><p style="color:var(--muted);max-width:600px;margin:0 auto">10 MCP tools. One API key. Infinite knowledge. Agents connect and research autonomously — no browser needed.</p></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p><p style="margin-top:8px">AI Research Engine v2.0 · Self-hosted · No KYC · Bitcoin Only</p></div></div></body></html>"""
|
||||
|
||||
PRICING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Pricing — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Simple Bitcoin Pricing</h2><p>No subscriptions. No KYC. Pay with Bitcoin, get API calls.</p></div><div style="max-width:500px;margin:0 auto"><div class="card" style="text-align:center;border-color:var(--accent);border-width:2px"><h3>🚀 Researcher Tier</h3><div class="price">$5<span> USD</span></div><p style="color:var(--muted);margin-bottom:16px">in Bitcoin · Lightning ⚡</p><ul class="feature-list" style="text-align:left;max-width:300px;margin:0 auto"><li>5 API calls</li><li>Full access to all 10 tools</li><li>Search, crawl, research, report</li><li>No expiration</li><li>No KYC required</li></ul><div style="margin-top:24px"><p style="color:var(--muted);font-size:0.85rem;margin-bottom:12px">👇 Sign up first, then buy from your dashboard</p><a href="/signup" class="btn btn-gold" style="width:100%">Sign Up & Get Your API Key</a></div></div></div><div class="grid" style="max-width:800px;margin:40px auto"><div class="card"><h3>⚡ Lightning Fast</h3><p>Payments settle in seconds. Your API calls are credited instantly.</p></div><div class="card"><h3>🔑 Bring Your Own Key</h3><p>Use your API key in any MCP client, script, or AI agent.</p></div><div class="card"><h3>📊 Track Usage</h3><p>Real-time dashboard shows your remaining calls and history.</p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p></div></div></body></html>"""
|
||||
|
||||
SIGNUP_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Sign Up — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Get Your API Key</h2><p>No KYC. Just an email. Your key is generated instantly.</p></div><div class="card" style="max-width:500px;margin:0 auto"><input type="email" id="email" placeholder="you@email.com" style="font-size:1.1rem"><button class="btn btn-primary" onclick="signup()" style="width:100%;font-size:1.1rem">Generate My API Key</button><div id="result" style="margin-top:20px;display:none"><p style="color:var(--green);margin-bottom:8px">✅ Your API key is ready:</p><pre id="apikey" style="cursor:pointer;word-break:break-all" onclick="copyKey()"></pre><p style="color:var(--muted);font-size:0.85rem;margin-top:8px">Click key to copy. Store it safely. No password reset — it's yours.</p><p style="color:var(--muted);font-size:0.85rem">Next: <a href="/pricing" style="color:var(--accent2)">buy API calls</a> or <a href="/dashboard" style="color:var(--accent2)">go to dashboard</a></p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
|
||||
async function signup(){{const e=document.getElementById('email').value;if(!e)return;const r=await fetch('/api/signup',{{method:'POST',headers:{{'Content-Type':'application/json'}},body:JSON.stringify({{email:e}})}});const d=await r.json();document.getElementById('result').style.display='block';document.getElementById('apikey').textContent=d.api_key||d.error}}
|
||||
function copyKey(){{const t=document.getElementById('apikey').textContent;navigator.clipboard.writeText(t);alert('API key copied!')}}
|
||||
</script></body></html>"""
|
||||
|
||||
DASHBOARD_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Dashboard — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero" style="padding:40px 0"><h2>Dashboard</h2><p>Your account, usage, and API access.</p></div><div id="login-section"><div class="card" style="max-width:500px;margin:0 auto"><h3>🔑 Enter Your API Key</h3><input type="text" id="apikey-input" placeholder="sk-..." style="font-family:monospace"><button class="btn btn-primary" onclick="login()" style="width:100%">View My Dashboard</button></div></div><div id="dashboard-section" style="display:none"><div class="grid" id="stats"></div><div class="card" style="margin-bottom:20px"><h3>⚡ Buy API Calls</h3><p style="color:var(--muted);margin-bottom:16px">$5 Bitcoin = 5 API calls. Pay with Lightning.</p><button class="btn btn-gold" onclick="buyCalls()">Pay $5 with Bitcoin ⚡</button><div id="invoice-result" style="margin-top:16px"></div></div><div class="card"><h3>📋 Quick Reference</h3><pre style="font-size:0.8rem"># Use your key in any HTTP request:
|
||||
curl -H "X-API-Key: YOUR_KEY" \\
|
||||
"http://10.30.20.249:8000/api/search?q=your+query"
|
||||
|
||||
# Or as a query parameter:
|
||||
curl "http://10.30.20.249:8000/api/search?q=test&api_key=YOUR_KEY"</pre></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
|
||||
let apiKey='';
|
||||
async function login(){{apiKey=document.getElementById('apikey-input').value;if(!apiKey)return;loadDashboard()}}
|
||||
async function loadDashboard(){{try{{const r=await fetch('/api/my-usage?api_key='+apiKey);const d=await r.json();document.getElementById('login-section').style.display='none';document.getElementById('dashboard-section').style.display='block';let h='<div class="card"><h3>📧 '+d.email+'</h3><div class="price" style="font-size:2rem">'+d.calls_remaining+'<span> calls left</span></div><p style="color:var(--muted)">'+d.total_calls+' total · since '+d.created_at+'</p></div>';h+='<div class="card"><h3>🔑 API Key</h3><pre style="word-break:break-all;cursor:pointer" onclick="navigator.clipboard.writeText(\''+d.api_key+'\')">'+d.api_key+'</pre><p style="color:var(--muted);font-size:0.8rem">Click to copy</p></div>';if(d.usage_by_tool){{h+='<div class="card"><h3>📊 Usage by Tool</h3>';for(const t of d.usage_by_tool)h+='<p style="display:flex;justify-content:space-between"><span>'+t.tool_name+'</span><span style="color:var(--accent2)">'+t.cnt+' calls</span></p>';h+='</div>'}}if(d.is_admin)h+='<div class="card" style="border-color:var(--gold)"><h3>👑 Admin Account</h3><p style="color:var(--gold)">Unlimited calls · Full system access</p></div>';document.getElementById('stats').innerHTML=h}}catch(e){{alert('Invalid API key')}}}}
|
||||
async function buyCalls(){{document.getElementById('invoice-result').innerHTML='<p style="color:var(--gold)">Creating invoice...</p>';const r=await fetch('/api/create-invoice?api_key='+apiKey,{{method:'POST'}});const d=await r.json();if(d.checkout_url){{document.getElementById('invoice-result').innerHTML='<a href="'+d.checkout_url+'" target="_blank" class="btn btn-gold" style="width:100%">Pay '+d.amount+' with Bitcoin →</a><p style="color:var(--muted);margin-top:8px;font-size:0.85rem">Opens BTCPay checkout. Refresh after payment.</p>'}}else{{document.getElementById('invoice-result').innerHTML='<p style="color:var(--red)">Error: '+JSON.stringify(d)+'</p>'}}}}
|
||||
</script></body></html>"""
|
||||
|
||||
|
||||
# ── Startup ──────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
61
server.py
61
server.py
@@ -1,110 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AI Research Engine — Thin MCP Proxy
|
||||
Runs on MacBook. Forwards all tool calls to the CT 145 backend.
|
||||
Minimal resource usage — all heavy lifting on Proxmox.
|
||||
"""
|
||||
"""AI Research Engine — Thin MCP Proxy. Admin key for backend auth."""
|
||||
|
||||
import json
|
||||
import httpx
|
||||
from mcp.server import FastMCP
|
||||
|
||||
BACKEND_URL = "http://10.30.20.249:8000"
|
||||
ADMIN_KEY = "sk-admin-unlimited-2026"
|
||||
client = httpx.Client(timeout=120.0)
|
||||
|
||||
mcp = FastMCP(
|
||||
"ai-research-engine",
|
||||
instructions="""
|
||||
AI Research Engine — private knowledge acquisition system.
|
||||
|
||||
search_web(query) — Full-text search across indexed documents
|
||||
semantic_search(query) — Find documents by meaning (vector search)
|
||||
crawl_url(url) — Crawl a URL into the index
|
||||
crawl_topic(topic) — Discover and crawl sources for a topic
|
||||
research_topic(topic) — Full pipeline: discover → crawl → summarize
|
||||
retrieve_document(url) — Get full content of an indexed document
|
||||
summarize_sources(urls, instruction) — AI summary of multiple sources
|
||||
extract_information(url, schema) — Structured data extraction
|
||||
create_report(topic, sources) — Generate comprehensive research report
|
||||
index_status() — System health and stats
|
||||
""",
|
||||
)
|
||||
|
||||
def _get(path: str) -> dict:
|
||||
r = client.get(f"{BACKEND_URL}{path}")
|
||||
sep = "&" if "?" in path else "?"
|
||||
r = client.get(f"{BACKEND_URL}{path}{sep}api_key={ADMIN_KEY}")
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
mcp = FastMCP("ai-research-engine", instructions="AI Research Engine — private knowledge acquisition. 10 tools.")
|
||||
|
||||
@mcp.tool()
|
||||
def search_web(query: str, category: str = "", limit: int = 10) -> str:
|
||||
"""Full-text search across indexed documents. Find by keywords, titles, content."""
|
||||
"""Full-text search across indexed documents."""
|
||||
r = _get(f"/api/search?q={query}&category={category}&limit={limit}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def semantic_search(query: str, limit: int = 10) -> str:
|
||||
"""Search by meaning using vector embeddings. Finds conceptually related docs."""
|
||||
"""Search by meaning using vector embeddings."""
|
||||
r = _get(f"/api/semantic-search?q={query}&limit={limit}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def crawl_url(url: str, depth: int = 1) -> str:
|
||||
"""Crawl a URL. depth: 0=just this page, 1=+linked pages."""
|
||||
"""Crawl a URL and index it."""
|
||||
r = _get(f"/api/crawl?url={url}&depth={depth}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def crawl_topic(topic: str, max_urls: int = 20) -> str:
|
||||
"""Discover and crawl sources for a topic using YaCy."""
|
||||
"""Discover and crawl sources for a topic."""
|
||||
r = _get(f"/api/crawl-topic?topic={topic}&max_urls={max_urls}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def research_topic(topic: str) -> str:
|
||||
"""Full research pipeline: keyword search → semantic search → crawl new sources → AI summary."""
|
||||
"""Full pipeline: search → crawl → AI summary."""
|
||||
r = _get(f"/api/research?topic={topic}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def retrieve_document(url: str) -> str:
|
||||
"""Get full indexed content of a document by URL."""
|
||||
"""Get full indexed content of a document."""
|
||||
r = _get(f"/api/document?url={url}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def summarize_sources(urls: str, instruction: str = "Summarize key points") -> str:
|
||||
"""Summarize multiple URLs using local LLM. urls: comma-separated."""
|
||||
"""AI summary of multiple URLs. urls: comma-separated."""
|
||||
r = _get(f"/api/summarize?urls={urls}&instruction={instruction}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def extract_information(url: str, schema: str = "company names, products, prices, specifications") -> str:
|
||||
"""Extract structured information from a document using LLM."""
|
||||
def extract_information(url: str, schema: str = "company names, products, prices") -> str:
|
||||
"""Extract structured data from a document using LLM."""
|
||||
r = _get(f"/api/extract?url={url}&schema={schema}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def create_report(topic: str, sources: str = "") -> str:
|
||||
"""Generate a comprehensive research report. sources: optional comma-separated URLs."""
|
||||
"""Generate comprehensive research report."""
|
||||
r = _get(f"/api/report?topic={topic}&sources={sources}")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def index_status() -> str:
|
||||
"""Check health of all backend services: OpenSearch, Qdrant, YaCy, Ollama."""
|
||||
"""System health + business stats."""
|
||||
r = _get("/api/status")
|
||||
return json.dumps(r, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mcp.run(transport="stdio")
|
||||
|
||||
Reference in New Issue
Block a user