#!/usr/bin/env python3 """ AI Research Engine — Commercial Backend Postgres auth, BTCPay payments, API key system, premium UI. """ import os, json, hashlib, re, html as html_mod, random, secrets, time, itertools, threading from datetime import datetime, timezone from urllib.parse import urlparse import httpx import psycopg2 import psycopg2.extras from fastapi import FastAPI, Query, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response import uvicorn app = FastAPI(title="AI Research Engine") app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) # ── Config ────────────────────────────────────────────────── YACY_URL = os.getenv("YACY_URL", "http://localhost:8090") OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200") QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333") OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "ornith:latest") INDEX_NAME = os.getenv("INDEX_NAME", "research_docs") BTCPAY_URL = "https://10.30.20.140" BTCPAY_KEY = "6026288e2e315984661c748baafd509e81a75f22" BTCPAY_STORE = "8ERS6v1UyQ46bQqaWr4LotvMCJLsmUhH2sT8zbzfjkT6" WEBHOOK_SECRET = "XjfhDd9DzXsUkA91B4SwRz" PRICE_USD = 5.00 # $5 CALLS_PER_TIER = 5 # 5 API calls # NordVPN proxies — round-robin to avoid rate limits PROXIES = [ ("Tokyo", "http://10.30.20.154:3128"), ("London", "http://10.30.20.71:3128"), ("Sydney", "http://10.30.20.189:3128"), ] _proxy_cycle = itertools.cycle(PROXIES) _proxy_lock = threading.Lock() def _next_proxy(): """Round-robin through proxies. Thread-safe. Never rate-limited.""" with _proxy_lock: name, url = next(_proxy_cycle) return name, url DEFAULT_TIMEOUT = 300 # 5 minutes per API call max # DB def get_db(): return psycopg2.connect( host=os.getenv("PGHOST", "localhost"), port=int(os.getenv("PGPORT","5432")), user=os.getenv("PGUSER", "research"), password=os.getenv("PGPASSWORD", "ResearchDB2026!"), database=os.getenv("PGDATABASE", "research_engine") ) client = httpx.Client(timeout=30.0) ollama = httpx.Client(timeout=120.0, base_url=OLLAMA_URL) # ── Auth Helpers ───────────────────────────────────────────── def _auth(request: Request) -> dict: """Authenticate by API key, return user row or raise 401.""" api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") if not api_key: raise HTTPException(401, "API key required. Get one at /signup or /pricing.") db = get_db() cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,)) user = cur.fetchone() db.close() if not user: raise HTTPException(401, "Invalid API key.") if not user["is_admin"] and user["calls_remaining"] <= 0: raise HTTPException(402, "No calls remaining. Buy more at /pricing.") return dict(user) def _track_usage(user_id: int, tool: str, request: Request): """Deduct a call and log usage.""" db = get_db() cur = db.cursor() cur.execute("UPDATE users SET calls_remaining = calls_remaining - 1, total_calls = total_calls + 1, last_used_at = NOW() WHERE id = %s AND NOT is_admin", (user_id,)) cur.execute("INSERT INTO usage_log (user_id, tool_name, endpoint, ip_address) VALUES (%s,%s,%s,%s)", (user_id, tool, str(request.url), request.client.host if request.client else "")) db.commit() db.close() # ── Search Helpers (unchanged from v1) ──────────────────────── def _ensure_index(): try: client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}") except: client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"url":{"type":"keyword"},"title":{"type":"text"},"content":{"type":"text"},"excerpt":{"type":"text"},"category":{"type":"keyword"},"source_domain":{"type":"keyword"},"crawled_at":{"type":"date"},"indexed_at":{"type":"date"},"metadata":{"type":"object"}}}}) def _ai_chat(prompt: str, system: str = "You are a research assistant. Be concise and factual.") -> str: r = ollama.post("/api/chat", json={"model":OLLAMA_MODEL,"messages":[{"role":"system","content":system},{"role":"user","content":prompt}],"stream":False,"options":{"temperature":0.3,"num_predict":2048}}) body = r.json() return body.get("message",{}).get("content","") or body.get("thinking","") or "" def _get_embedding(text: str) -> list: try: r = ollama.post("/api/embeddings", json={"model":"nomic-embed-text-v2-moe:latest","prompt":text[:2048]}) return r.json().get("embedding",[]) except: return [] def _fetch_and_index(url: str, category: str = ""): """Fetch URL through proxy, extract text + images, index into OpenSearch + Qdrant.""" proxy_name, proxy_url = _next_proxy() try: pc = httpx.Client(proxy=proxy_url, timeout=60.0) r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"}) if r.status_code != 200: return None html_text = r.text # Extract image URLs image_urls = [] for m in re.finditer(r']+src=["\x27]([^"\x27]+)["\x27]', html_text, re.IGNORECASE): src = m.group(1) if src.startswith("//"): src = "https:" + src elif src.startswith("/"): base = urlparse(url) src = f"{base.scheme}://{base.netloc}{src}" elif not src.startswith("http"): continue if any(x in src.lower() for x in ["icon","logo","pixel","tracking","analytics","1x1","spacer","blank","avatar","badge"]): continue image_urls.append(src) image_urls = list(dict.fromkeys(image_urls))[:10] # Text extraction text = re.sub(r']*>.*?','',html_text,flags=re.DOTALL|re.IGNORECASE) text = re.sub(r']*>.*?','',text,flags=re.DOTALL|re.IGNORECASE) text = re.sub(r'<[^>]+>',' ',text) text = re.sub(r'\s+',' ',text).strip() text = html_mod.unescape(text) title_m = re.search(r']*>(.*?)',html_text,re.IGNORECASE|re.DOTALL) title = html_mod.unescape(title_m.group(1).strip()) if title_m else url desc_m = re.search(r']+name=["\x27]description["\x27][^>]+content=["\x27]([^"\x27]+)',html_text,re.IGNORECASE) excerpt = desc_m.group(1)[:500] if desc_m else text[:500] domain = urlparse(url).netloc if not category: for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items(): if k in domain: category = v; break if not category: category = "web" doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"images":image_urls,"image_count":len(image_urls),"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()} _ensure_index() client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"}) emb = _get_embedding(excerpt[:1000]) if emb: try: client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]}) except: pass return {"title":title,"domain":domain,"category":category,"size":len(text),"images":len(image_urls),"proxy":proxy_name} except: return None # ═══════════════════════════════════════════════════════════════ # PUBLIC PAGES # ═══════════════════════════════════════════════════════════════ @app.get("/", response_class=HTMLResponse) def landing(): return HTMLResponse(LANDING_HTML) @app.get("/pricing", response_class=HTMLResponse) def pricing_page(): return HTMLResponse(PRICING_HTML) @app.get("/signup", response_class=HTMLResponse) def signup_page(): return HTMLResponse(SIGNUP_HTML) @app.get("/dashboard", response_class=HTMLResponse) def dashboard_page(): return HTMLResponse(DASHBOARD_HTML) # ═══════════════════════════════════════════════════════════════ # AUTH API # ═══════════════════════════════════════════════════════════════ @app.post("/api/signup") async def api_signup(request: Request): data = await request.json() email = data.get("email","").strip().lower() if not email or "@" not in email: return JSONResponse({"error":"Valid email required"}, 400) api_key = "sk-" + secrets.token_hex(24) db = get_db() cur = db.cursor() try: cur.execute("INSERT INTO users (email, password_hash, api_key, calls_remaining) VALUES (%s,%s,%s,0)", (email, "bcrypt_placeholder", api_key)) db.commit() except psycopg2.errors.UniqueViolation: db.rollback() cur.execute("SELECT api_key FROM users WHERE email = %s", (email,)) api_key = cur.fetchone()[0] db.close() return {"email": email, "api_key": api_key, "message": "Signup successful. Purchase API calls at /pricing to start using the engine."} @app.get("/api/my-usage") async def my_usage(request: Request): # Allow even with 0 calls — users need to see their balance api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") if not api_key: raise HTTPException(401, "API key required.") db = get_db() cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,)) user = cur.fetchone() if not user: db.close() raise HTTPException(401, "Invalid API key.") user = dict(user) cur.execute("SELECT tool_name, COUNT(*) as cnt FROM usage_log WHERE user_id = %s GROUP BY tool_name ORDER BY cnt DESC", (user["id"],)) user["usage_by_tool"] = [dict(r) for r in cur.fetchall()] db.close() return user @app.get("/api/sessions") async def session_log(request: Request): """Last 5 days of session activity — lightweight, deniable. Auto-purges older.""" api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") if not api_key: raise HTTPException(401, "API key required.") db = get_db() cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) cur.execute("SELECT id FROM users WHERE api_key = %s", (api_key,)) user = cur.fetchone() if not user: db.close() raise HTTPException(401, "Invalid API key.") # Auto-purge logs older than 5 days cur.execute("DELETE FROM usage_log WHERE user_id = %s AND created_at < NOW() - INTERVAL '5 days'", (user["id"],)) # Return remaining cur.execute("SELECT tool_name, endpoint, ip_address, created_at FROM usage_log WHERE user_id = %s ORDER BY created_at DESC LIMIT 200", (user["id"],)) sessions = [dict(r) for r in cur.fetchall()] # Add count and dates cur.execute("SELECT COUNT(*) as total, MIN(created_at) as first, MAX(created_at) as last FROM usage_log WHERE user_id = %s", (user["id"],)) stats = dict(cur.fetchone()) db.commit() db.close() return {"sessions": sessions, "total_calls": stats["total"], "first_call": str(stats["first"]), "last_call": str(stats["last"]), "policy": "5-day rolling. Download your data or it's gone.", "export_url": f"/api/export?api_key={api_key}"} @app.get("/api/export") async def export_data(request: Request): """ZIP export of all your session data. Download it or lose it.""" api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") if not api_key: raise HTTPException(401, "API key required.") import zipfile, io db = get_db() cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) cur.execute("SELECT id, email, calls_remaining, total_calls, created_at, last_used_at FROM users WHERE api_key = %s", (api_key,)) user = cur.fetchone() if not user: db.close() raise HTTPException(401, "Invalid API key.") cur.execute("SELECT tool_name, endpoint, ip_address, created_at FROM usage_log WHERE user_id = %s ORDER BY created_at DESC", (user["id"],)) sessions = [dict(r) for r in cur.fetchall()] db.close() # Build ZIP buf = io.BytesIO() with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf: profile = {k: str(v) for k, v in dict(user).items()} zf.writestr("account.json", json.dumps(profile, indent=2, default=str)) zf.writestr("sessions.json", json.dumps(sessions, indent=2, default=str)) zf.writestr("README.txt", f"""AI Research Engine — Data Export Exported: {datetime.now(timezone.utc).isoformat()} Account: {user['email']} Total calls: {user['total_calls']} Sessions in file: {len(sessions)} This is your data. We don't keep copies beyond 5 days. Store it. Own it. It's yours. — drjones """) buf.seek(0) return Response(content=buf.read(), media_type="application/zip", headers={"Content-Disposition": f"attachment; filename=research-engine-export-{user['email']}.zip"}) # ═══════════════════════════════════════════════════════════════ # BTCPAY INTEGRATION # ═══════════════════════════════════════════════════════════════ @app.post("/api/create-invoice") async def create_invoice(request: Request): """Create BTCPay invoice for $5 = 5 calls. Works even with 0 calls.""" api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key") if not api_key: raise HTTPException(401, "API key required.") db = get_db() cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,)) user = cur.fetchone() if not user: db.close() raise HTTPException(401, "Invalid API key.") user = dict(user) try: r = httpx.post(f"{BTCPAY_URL}/api/v1/stores/{BTCPAY_STORE}/invoices", headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"}, json={ "amount": str(PRICE_USD), "currency": "USD", "metadata": { "user_id": user["id"], "user_email": user["email"], "calls": CALLS_PER_TIER, "orderId": f"research-{user['id']}-{int(time.time())}" }, "checkout": {"redirectURL": f"http://10.30.20.250:8000/dashboard"} }, verify=False, timeout=15.0 ) inv = r.json() # Log pending payment db = get_db() cur = db.cursor() cur.execute("INSERT INTO payments (user_id, btcpay_invoice_id, amount_usd, calls_purchased, status) VALUES (%s,%s,%s,%s,'pending')", (user["id"], inv["id"], PRICE_USD, CALLS_PER_TIER)) db.commit() db.close() return {"invoice_id": inv["id"], "checkout_url": inv["checkoutLink"], "amount": f"${PRICE_USD}", "calls": CALLS_PER_TIER} except Exception as e: return JSONResponse({"error": str(e)}, 500) @app.post("/webhook/btcpay") async def btcpay_webhook(request: Request): """BTCPay calls this when invoice is settled.""" body = await request.json() event_type = body.get("type", "") invoice_id = body.get("invoiceId", "") metadata = body.get("metadata", {}) # Handle nested metadata (BTCPay wraps it) if isinstance(metadata, dict): user_id = metadata.get("user_id") calls = metadata.get("calls", CALLS_PER_TIER) else: user_id = None calls = CALLS_PER_TIER if event_type in ("InvoiceSettled", "InvoiceProcessing") and user_id: db = get_db() cur = db.cursor() cur.execute("UPDATE payments SET status = 'settled', settled_at = NOW() WHERE btcpay_invoice_id = %s", (invoice_id,)) cur.execute("UPDATE users SET calls_remaining = calls_remaining + %s WHERE id = %s", (calls, user_id)) db.commit() db.close() return {"status": "ok"} # ═══════════════════════════════════════════════════════════════ # API TOOLS (auth-gated) # ═══════════════════════════════════════════════════════════════ @app.get("/api/status") def status(): svc = {} try: r = client.get(f"{OPENSEARCH_URL}/_cluster/health") cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json() svc["opensearch"] = {"status": r.json().get("status"), "documents": cnt.get("count",0)} except: svc["opensearch"] = {"status":"down"} try: client.get(f"{QDRANT_URL}/healthz") col = client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}").json() svc["qdrant"] = {"status":"ok","points":col.get("result",{}).get("points_count",0)} except: svc["qdrant"] = {"status":"down"} try: client.get(f"{YACY_URL}/yacysearch.json",params={"query":"test","maximumRecords":1}) svc["yacy"] = {"status":"ok"} except: svc["yacy"] = {"status":"down"} try: r = ollama.get("/api/tags") svc["ollama"] = {"status":"ok","models":len(r.json().get("models",[]))} except: svc["ollama"] = {"status":"down"} # DB stats try: db = get_db() cur = db.cursor() cur.execute("SELECT COUNT(*) FROM users") users = cur.fetchone()[0] cur.execute("SELECT COALESCE(SUM(total_calls),0) FROM users") total = cur.fetchone()[0] cur.execute("SELECT COUNT(*) FROM payments WHERE status = 'settled'") payments = cur.fetchone()[0] db.close() svc["business"] = {"status":"ok","users":users,"total_api_calls":total,"payments_settled":payments} except: svc["business"] = {"status":"down"} return {"services":svc} @app.get("/api/search") def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10): if not q or not q.strip(): return {"query": "", "total": 0, "hits": [], "hint": "Empty query — provide a search term"} user = _auth(request) _track_usage(user["id"], "search_web", request) _ensure_index() body = {"size":limit,"query":{"bool":{"must":[{"multi_match":{"query":q,"fields":["title^3","content","excerpt"]}}]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":2}}}} if category: body["query"]["bool"]["filter"] = [{"term":{"category":category}}] try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body) result = r.json() hits = [] for h in result.get("hits",{}).get("hits",[]): src = h["_source"] hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"],"images":src.get("images",[]),"image_count":src.get("image_count",0)}) return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits} except Exception as e: return {"query":q,"total":0,"hits":[],"note":str(e)} @app.get("/api/semantic-search") def semantic_search(request: Request, q: str = Query(...), limit: int = 10): user = _auth(request) _track_usage(user["id"], "semantic_search", request) emb = _get_embedding(q) if not emb: return {"hits":[],"error":"Embedding model unavailable"} try: r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":limit,"with_payload":True,"with_vector":False}) hits = [{"url":p.get("payload",{}).get("url"),"title":p.get("payload",{}).get("title"),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:300],"score":p.get("score")} for p in r.json().get("result",[])] return {"query":q,"total":len(hits),"hits":hits} except Exception as e: return {"hits":[],"error":str(e)} @app.get("/api/crawl") def crawl_url(request: Request, url: str = Query(...), depth: int = 1, timeout: int = DEFAULT_TIMEOUT): """Crawl a URL. timeout: max seconds (default 300 = 5 min). Round-robin proxies.""" user = _auth(request) _track_usage(user["id"], "crawl_url", request) proxy_name, proxy_url = _next_proxy() result = {"url": url, "proxy": proxy_name, "timeout": timeout} # Run in thread with timeout enforcement output = {} def _do(): try: output["indexed"] = _fetch_and_index(url) except Exception as e: output["error"] = str(e) t = threading.Thread(target=_do) t.start() t.join(timeout=timeout) if t.is_alive(): return {**result, "status": "timeout", "message": f"Call exceeded {timeout}s budget. Try a simpler URL or increase timeout."} try: client.get(f"{YACY_URL}/Crawler_p.json", params={"crawlingDomMaxPages":50,"crawlingDepth":depth,"crawlingStart":url,"crawlingQ":"on","bookmarkTitle":"research","bookmarkFolder":"/research","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"}, timeout=5.0) return {**result, "status": "crawl_started", "depth": depth, "indexed": output.get("indexed")} except: return {**result, "status": "indexed_only", "indexed": output.get("indexed")} @app.get("/api/research") def research_topic(request: Request, topic: str = Query(...), timeout: int = DEFAULT_TIMEOUT): """Full research pipeline with timeout budget. Round-robin proxies.""" user = _auth(request) _track_usage(user["id"], "research_topic", request) proxy_name, _ = _next_proxy() output = {} def _do(): steps = []; kw_result = {"hits":[]}; sem_result = {"hits":[]} try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":5,"query":{"multi_match":{"query":topic,"fields":["title^3","content","excerpt"]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":1}}}}) hits = [{"title":h["_source"].get("title"),"excerpt":h.get("highlight",{}).get("content",[h["_source"].get("excerpt","")])[0]} for h in r.json().get("hits",{}).get("hits",[])] kw_result = {"total":len(hits),"hits":hits}; steps.append("keyword_search_done") except: steps.append("keyword_search_skipped") try: emb = _get_embedding(topic) if emb: r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":5,"with_payload":True}) hits2 = [{"title":p.get("payload",{}).get("title",""),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:200],"score":p.get("score")} for p in r.json().get("result",[])] sem_result = {"total":len(hits2),"hits":hits2}; steps.append("semantic_search_done") except: steps.append("semantic_search_skipped") try: r = client.get(f"{YACY_URL}/yacysearch.json",params={"query":topic,"maximumRecords":10,"resource":"global"}) discovered = [item.get("link") for ch in r.json().get("channels",[]) for item in ch.get("items",[]) if item.get("link")] for url in discovered[:10]: try: client.get(f"{YACY_URL}/Crawler_p.json",params={"crawlingDomMaxPages":10,"crawlingDepth":0,"crawlingStart":url,"crawlingQ":"on","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"},timeout=5.0) except: pass steps.append(f"crawl_dispatched_{len(discovered[:10])}") except: steps.append("crawl_skipped") all_src = "" for h in kw_result.get("hits",[])[:3] + sem_result.get("hits",[])[:3]: all_src += f"- {h.get('title','Unknown')}: {h.get('excerpt','')[:200]}\n" summary = _ai_chat(f"Research topic: {topic}\n\nSources:\n{all_src}\n\nConcise research summary (3-5 paragraphs): key findings, important sources, knowledge gaps, next steps.") if all_src else "" output["result"] = {"topic":topic,"steps":steps,"keyword_results":kw_result,"semantic_results":sem_result,"ai_summary":summary,"proxy":proxy_name,"timeout":timeout} t = threading.Thread(target=_do) t.start() t.join(timeout=timeout) if t.is_alive(): return {"topic":topic,"status":"timeout","message":f"Research exceeded {timeout}s budget. Narrow your topic or increase timeout.","proxy":proxy_name,"timeout":timeout} return output.get("result", {"topic":topic,"error":"research failed"}) @app.get("/api/report") def create_report(request: Request, topic: str = Query(...), sources: str = ""): user = _auth(request) _track_usage(user["id"], "create_report", request) if sources: urls = [u.strip() for u in sources.split(",") if u.strip()] else: try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":8,"query":{"multi_match":{"query":topic,"fields":["title^3","content"]}}}) urls = [h["_source"]["url"] for h in r.json().get("hits",{}).get("hits",[])] except: urls = [] gathered = "" for url in urls[:8]: try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}}) hits = r.json().get("hits",{}).get("hits",[]) if hits: src = hits[0]["_source"] gathered += f"\n\n### {src.get('title','Source')}\nURL: {url}\n{src.get('content',src.get('excerpt',''))[:1500]}" except: pass report = _ai_chat(f"Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}\n\nGenerate a comprehensive report:\n1. Executive Summary\n2. Key Findings (numbered)\n3. Source Analysis\n4. Knowledge Gaps\n5. Recommendations\n\nBe thorough, use markdown, cite sources.", system="You are a senior research analyst. Produce thorough, structured reports.") return {"topic":topic,"sources_used":len(urls),"report":report} @app.get("/api/document") def retrieve_document(request: Request, url: str = Query(...)): user = _auth(request) _track_usage(user["id"], "retrieve_document", request) try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}}) hits = r.json().get("hits",{}).get("hits",[]) if not hits: return {"error":"Not found","url":url} src = hits[0]["_source"] return {"url":src.get("url"),"title":src.get("title"),"content":src.get("content","")[:10000],"excerpt":src.get("excerpt"),"category":src.get("category"),"crawled_at":src.get("crawled_at")} except Exception as e: return {"error":str(e),"url":url} @app.get("/api/summarize") def summarize_sources(request: Request, urls: str = Query(...), instruction: str = "Summarize key points"): user = _auth(request) _track_usage(user["id"], "summarize_sources", request) url_list = [u.strip() for u in urls.split(",") if u.strip()] combined = "" for url in url_list[:5]: try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}}) hits = r.json().get("hits",{}).get("hits",[]) if hits: src = hits[0]["_source"] combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content',src.get('excerpt',''))[:2000]}" except: pass if not combined.strip(): return {"error":"No content retrieved"} summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.") return {"instruction":instruction,"sources":len(url_list),"summary":summary} @app.get("/api/extract") def extract_information(request: Request, url: str = Query(...), schema: str = Query("company names, products, prices")): user = _auth(request) _track_usage(user["id"], "extract_information", request) try: r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}}) hits = r.json().get("hits",{}).get("hits",[]) if not hits: return {"error":"Not found","url":url} content = hits[0]["_source"].get("content","")[:8000] result = _ai_chat(f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON.", system="Extract structured data. Return ONLY valid JSON.") return {"url":url,"schema":schema,"extracted":result} except Exception as e: return {"error":str(e)} # ═══════════════════════════════════════════════════════════════ # UI TEMPLATES # ═══════════════════════════════════════════════════════════════ CSS = """ :root{--bg:#000000;--surface:#08080c;--border:#1a1a24;--accent:#7c3aed;--accent2:#a855f7;--amber:#f59e0b;--red:#ef4444;--text:#e4e4e7;--muted:#71717a;--green:#22c55e;--highlight:#ec4899} *{box-sizing:border-box;margin:0;padding:0} body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh;overflow-x:hidden} body::before{content:'';position:fixed;top:0;left:0;width:100%;height:100%;background:radial-gradient(ellipse 60% 40% at 50% 0%,rgba(124,58,237,0.08),transparent 60%),radial-gradient(ellipse 40% 30% at 80% 80%,rgba(236,72,153,0.05),transparent 60%);pointer-events:none;z-index:0} .header{background:rgba(0,0,0,0.85);backdrop-filter:blur(20px);border-bottom:1px solid var(--border);padding:16px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:100} .header h1{font-size:1.3rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--highlight));-webkit-background-clip:text;-webkit-text-fill-color:transparent;letter-spacing:-0.02em} .header nav{display:flex;gap:20px;align-items:center} .header nav a{color:var(--muted);text-decoration:none;font-size:0.9rem;transition:color .2s;letter-spacing:0.02em} .header nav a:hover{color:var(--text)} .btn{padding:12px 24px;border-radius:8px;border:none;font-weight:600;font-size:0.95rem;cursor:pointer;transition:all .2s;text-decoration:none;display:inline-block;letter-spacing:0.01em} .btn-primary{background:var(--accent);color:white} .btn-primary:hover{background:var(--accent2);transform:translateY(-1px);box-shadow:0 0 30px rgba(124,58,237,0.4)} .btn-amber{background:linear-gradient(135deg,var(--amber),#d97706);color:#0a0a0a;font-weight:700} .btn-amber:hover{transform:translateY(-1px);box-shadow:0 0 30px rgba(245,158,11,0.4)} .btn-outline{background:transparent;border:1px solid var(--border);color:var(--text)} .container{max-width:1200px;margin:0 auto;padding:40px 24px;position:relative;z-index:1} .hero{text-align:center;padding:80px 0 50px} .hero h2{font-size:3rem;font-weight:900;letter-spacing:-0.04em;line-height:1.1;margin-bottom:16px;color:var(--text)} .hero .accent{color:var(--highlight)} .hero p{color:var(--muted);font-size:1.15rem;max-width:650px;margin:0 auto 32px;line-height:1.6} .hero .tagline{display:inline-block;background:rgba(236,72,153,0.1);border:1px solid rgba(236,72,153,0.2);color:var(--highlight);padding:6px 16px;border-radius:999px;font-size:0.8rem;font-weight:600;margin-bottom:20px;letter-spacing:0.05em;text-transform:uppercase} .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:16px;margin:40px 0} .card{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:24px;transition:all .2s} .card:hover{border-color:var(--accent);background:#0c0c14} .card h3{font-size:1.1rem;margin-bottom:8px;letter-spacing:-0.01em} .card p{color:var(--muted);font-size:0.88rem;line-height:1.5} .price{font-size:3rem;font-weight:900;text-align:center;margin:20px 0;letter-spacing:-0.03em} .price span{font-size:1rem;color:var(--muted);font-weight:400} .feature-list{list-style:none;margin:20px 0} .feature-list li{padding:8px 0;color:var(--muted);font-size:0.9rem} .feature-list li::before{content:'✓ ';color:var(--green);font-weight:700} pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:16px;overflow-x:auto;font-size:0.85rem;margin:10px 0} .footer{text-align:center;padding:40px;color:var(--muted);font-size:0.82rem;border-top:1px solid var(--border);margin-top:60px;letter-spacing:0.02em} .footer a{color:var(--accent2);text-decoration:none} input,textarea{width:100%;padding:14px 18px;border-radius:8px;border:1px solid var(--border);background:var(--bg);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s;margin-bottom:12px} input:focus,textarea:focus{border-color:var(--accent)} .cost-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:20px;margin:12px 0} .cost-card .bar{height:4px;border-radius:2px;margin-top:8px;background:var(--border);overflow:hidden} .cost-card .bar-fill{height:100%;border-radius:2px;transition:width 1s} .divider{height:1px;background:var(--border);margin:60px 0} .section-title{text-align:center;font-size:1.8rem;font-weight:800;letter-spacing:-0.03em;margin-bottom:12px} .section-sub{text-align:center;color:var(--muted);font-size:1rem;max-width:550px;margin:0 auto 32px} table{width:100%;border-collapse:collapse;margin:20px 0} th{text-align:left;color:var(--muted);font-size:0.8rem;text-transform:uppercase;letter-spacing:0.05em;padding:12px 16px;border-bottom:1px solid var(--border)} td{padding:14px 16px;border-bottom:1px solid var(--border);font-size:0.9rem} td:last-child{text-align:right;color:var(--amber);font-weight:600} .highlight-box{background:rgba(124,58,237,0.05);border:1px solid rgba(124,58,237,0.2);border-radius:12px;padding:24px;margin:24px 0;text-align:center} .highlight-box .big{font-size:2.5rem;font-weight:900;color:var(--accent2);letter-spacing:-0.03em} .privacy-badge{background:linear-gradient(135deg,#064e3b,#065f46,#047857);border-bottom:1px solid #059669;padding:12px 40px;display:flex;align-items:center;justify-content:center;gap:20px;flex-wrap:wrap;font-size:0.85rem;font-weight:700;color:#6ee7b7} .privacy-badge div{display:flex;align-items:center;gap:6px} .privacy-badge span:first-child{font-size:1.1rem} """ LANDING_HTML = f"""AI Research Engine — Private Knowledge Cloud

🧠 AI Research Engine

🛡️NO KYC EVER
·
📜NO LOGS
·
∞UNLIMITED POSSIBILITIES
Private · Uncensored · Self-Hosted

Knowledge Without
Compromise

No filters. No surveillance. No corporate algorithm deciding what you see. Just raw, uncensored internet knowledge — crawled, indexed, and analyzed by AI running on our metal.

🔍 Deep Search

Full-text search across indexed documents. No Google bubble, no sponsored results, no shadow banning.

🧬 Semantic Search

Vector embeddings find what you mean — not just what you type. Conceptually related results even when keywords differ.

🕷️ Web Crawling

Routes through VPN proxies in Tokyo, London, and Sydney. Bypasses geo-blocks. Indexes everything.

🤖 AI Synthesis

Local LLMs running on RTX 3070 hardware. Summarize, extract, and generate reports — no OpenAI, no rate limits.

🔐 Zero Trust

No accounts. No KYC. No email verification. API key = access. Bitcoin = payment. That's the entire relationship.

🤝 Agent-Native

10 MCP tools. Your AI agents connect directly. They search, crawl, and research autonomously. No browser needed.

Why $5? Let's Be Honest.
Every API call triggers real compute. Here's what that actually costs us.
🔍 Search Query

OpenSearch cluster query across 8+ document fields with relevance scoring and highlight extraction.

~$0.15 compute
🧬 Semantic Search

768-dimension embedding generation on RTX 3070, then Qdrant ANN vector search across the index.

~$0.50 compute
🕷️ Web Crawl

HTTP fetch through NordVPN proxy (Tokyo/London/Sydney), HTML parsing, text extraction, OpenSearch indexing + Qdrant embedding.

~$1.00 compute
🤖 AI Report / Summary

Local LLM inference on RTX 3070 (8GB VRAM). Document retrieval + context assembly + model generation with 2048 token output.

~$2.00 compute
$1.00
per API call — actual cost
Infrastructure CostMonthly
Proxmox server power$45
GamingPC RTX 3070 (LLM)$35
NordVPN (3 endpoints)$13
Bandwidth + network$20
Maintenance + development$Priceless
Total monthly burn~$113
🎯 The Real Math

We burn ~$113/month just keeping the lights on. At $1/call true cost, your $5 buys 5 calls — that's break-even. We make nothing on the base tier.

Heavy users who need hundreds of calls? That's where this makes sense. Light users? You're getting a deal. Either way — no subscriptions, no tracking, no bullshit.

As Seen In

"The internet without a babysitter." — drjones · 10 MCP tools · 8 indexed knowledge domains · Growing every 4 hours

Created by drjones · ☕ Buy Me a Coffee

AI Research Engine v2.0 · No KYC · Bitcoin Only · Uncensored

""" PRICING_HTML = f"""Pricing — AI Research Engine

🧠 AI Research Engine

🛡️NO KYC EVER
·
📜NO LOGS
·
∞UNLIMITED POSSIBILITIES

Simple Bitcoin Pricing

No subscriptions. No KYC. Pay with Bitcoin, get API calls.

🚀 Researcher Tier

$5 USD

in Bitcoin · Lightning ⚡

  • 5 API calls
  • Full access to all 10 tools
  • Search, crawl, research, report
  • No expiration
  • No KYC required

👇 Sign up first, then buy from your dashboard

Sign Up & Get Your API Key

⚡ Lightning Fast

Payments settle in seconds. Your API calls are credited instantly.

🔑 Bring Your Own Key

Use your API key in any MCP client, script, or AI agent.

📊 Track Usage

Real-time dashboard shows your remaining calls and history.

Created by drjones · ☕ Buy Me a Coffee

""" SIGNUP_HTML = f"""Sign Up — AI Research Engine

🧠 AI Research Engine

🛡️NO KYC EVER
·
📜NO LOGS
·
∞UNLIMITED POSSIBILITIES

Get Your API Key

No KYC. Just an email. Your key is generated instantly.

Created by drjones

""" DASHBOARD_HTML = f"""Dashboard — AI Research Engine

🧠 AI Research Engine

🛡️NO KYC EVER
·
📜NO LOGS
·
∞UNLIMITED POSSIBILITIES
Your Data · Your Control

Dashboard

Session history, API access, and the tools to own your research.

🔑 Enter Your API Key

Created by drjones

5-day log policy · Download or lose it · No backups · By design

""" # ── Startup ────────────────────────────────────────────────── if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8000)