c29b9e1ba9
CT 144 (certus-workbrain) restored to 10.30.20.249 All configs updated: MCP proxy, backend, BTCPay webhook
518 lines
38 KiB
Python
518 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
AI Research Engine — Commercial Backend
|
|
Postgres auth, BTCPay payments, API key system, premium UI.
|
|
"""
|
|
|
|
import os, json, hashlib, re, html as html_mod, random, secrets, time
|
|
from datetime import datetime, timezone
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
from fastapi import FastAPI, Query, HTTPException, Request
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
|
|
import uvicorn
|
|
|
|
app = FastAPI(title="AI Research Engine")
|
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
|
|
|
# ── Config ──────────────────────────────────────────────────
|
|
YACY_URL = os.getenv("YACY_URL", "http://host.docker.internal:8090")
|
|
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://host.docker.internal:9200")
|
|
QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
|
|
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.30.20.186:11434")
|
|
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "ornith:latest")
|
|
INDEX_NAME = os.getenv("INDEX_NAME", "research_docs")
|
|
BTCPAY_URL = "https://10.30.20.140"
|
|
BTCPAY_KEY = "6026288e2e315984661c748baafd509e81a75f22"
|
|
BTCPAY_STORE = "8ERS6v1UyQ46bQqaWr4LotvMCJLsmUhH2sT8zbzfjkT6"
|
|
WEBHOOK_SECRET = "XjfhDd9DzXsUkA91B4SwRz"
|
|
PRICE_USD = 5.00 # $5
|
|
CALLS_PER_TIER = 5 # 5 API calls
|
|
|
|
# Proxies
|
|
PROXIES = [
|
|
"http://10.30.20.154:3128", # Tokyo
|
|
"http://10.30.20.71:3128", # London
|
|
"http://10.30.20.189:3128", # Sydney
|
|
]
|
|
|
|
# DB
|
|
def get_db():
|
|
return psycopg2.connect(
|
|
host="host.docker.internal", port=5432,
|
|
user="research", password="ResearchDB2026!",
|
|
database="research_engine"
|
|
)
|
|
|
|
client = httpx.Client(timeout=30.0)
|
|
ollama = httpx.Client(timeout=120.0, base_url=OLLAMA_URL)
|
|
|
|
# ── Auth Helpers ─────────────────────────────────────────────
|
|
|
|
def _auth(request: Request) -> dict:
|
|
"""Authenticate by API key, return user row or raise 401."""
|
|
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
|
if not api_key:
|
|
raise HTTPException(401, "API key required. Get one at /signup or /pricing.")
|
|
db = get_db()
|
|
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
|
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
|
user = cur.fetchone()
|
|
db.close()
|
|
if not user:
|
|
raise HTTPException(401, "Invalid API key.")
|
|
if not user["is_admin"] and user["calls_remaining"] <= 0:
|
|
raise HTTPException(402, "No calls remaining. Buy more at /pricing.")
|
|
return dict(user)
|
|
|
|
def _track_usage(user_id: int, tool: str, request: Request):
|
|
"""Deduct a call and log usage."""
|
|
db = get_db()
|
|
cur = db.cursor()
|
|
cur.execute("UPDATE users SET calls_remaining = calls_remaining - 1, total_calls = total_calls + 1, last_used_at = NOW() WHERE id = %s AND NOT is_admin", (user_id,))
|
|
cur.execute("INSERT INTO usage_log (user_id, tool_name, endpoint, ip_address) VALUES (%s,%s,%s,%s)",
|
|
(user_id, tool, str(request.url), request.client.host if request.client else ""))
|
|
db.commit()
|
|
db.close()
|
|
|
|
# ── Search Helpers (unchanged from v1) ────────────────────────
|
|
|
|
def _ensure_index():
|
|
try: client.get(f"{OPENSEARCH_URL}/{INDEX_NAME}")
|
|
except: client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}", json={"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"url":{"type":"keyword"},"title":{"type":"text"},"content":{"type":"text"},"excerpt":{"type":"text"},"category":{"type":"keyword"},"source_domain":{"type":"keyword"},"crawled_at":{"type":"date"},"indexed_at":{"type":"date"},"metadata":{"type":"object"}}}})
|
|
|
|
def _ai_chat(prompt: str, system: str = "You are a research assistant. Be concise and factual.") -> str:
|
|
r = ollama.post("/api/chat", json={"model":OLLAMA_MODEL,"messages":[{"role":"system","content":system},{"role":"user","content":prompt}],"stream":False,"options":{"temperature":0.3,"num_predict":2048}})
|
|
body = r.json()
|
|
return body.get("message",{}).get("content","") or body.get("thinking","") or ""
|
|
|
|
def _get_embedding(text: str) -> list:
|
|
try:
|
|
r = ollama.post("/api/embeddings", json={"model":"nomic-embed-text-v2-moe:latest","prompt":text[:2048]})
|
|
return r.json().get("embedding",[])
|
|
except: return []
|
|
|
|
def _fetch_and_index(url: str, category: str = ""):
|
|
proxy_url = random.choice(PROXIES)
|
|
try:
|
|
pc = httpx.Client(proxy=proxy_url, timeout=15.0)
|
|
r = pc.get(url, headers={"User-Agent":"Mozilla/5.0 (compatible; ResearchBot/1.0)"})
|
|
if r.status_code != 200: return None
|
|
html_text = r.text
|
|
text = re.sub(r'<script[^>]*>.*?</script>','',html_text,flags=re.DOTALL|re.IGNORECASE)
|
|
text = re.sub(r'<style[^>]*>.*?</style>','',text,flags=re.DOTALL|re.IGNORECASE)
|
|
text = re.sub(r'<[^>]+>',' ',text)
|
|
text = re.sub(r'\s+',' ',text).strip()
|
|
text = html_mod.unescape(text)
|
|
title_m = re.search(r'<title[^>]*>(.*?)</title>',html_text,re.IGNORECASE|re.DOTALL)
|
|
title = html_mod.unescape(title_m.group(1).strip()) if title_m else url
|
|
desc_m = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']+)',html_text,re.IGNORECASE)
|
|
excerpt = desc_m.group(1)[:500] if desc_m else text[:500]
|
|
domain = urlparse(url).netloc
|
|
if not category:
|
|
for k,v in {"wikipedia":"reference","github":"software","arxiv":"science","docs.":"documentation","blog.":"blog","news.":"news"}.items():
|
|
if k in domain: category = v; break
|
|
if not category: category = "web"
|
|
doc = {"url":url,"title":title,"content":text[:50000],"excerpt":excerpt[:1000],"category":category,"source_domain":domain,"crawled_at":datetime.now(timezone.utc).isoformat(),"indexed_at":datetime.now(timezone.utc).isoformat()}
|
|
_ensure_index()
|
|
client.put(f"{OPENSEARCH_URL}/{INDEX_NAME}/_doc/{hashlib.md5(url.encode()).hexdigest()}",json=doc,params={"refresh":"true"})
|
|
emb = _get_embedding(excerpt[:1000])
|
|
if emb:
|
|
try:
|
|
client.put(f"{QDRANT_URL}/collections/{INDEX_NAME}/points",json={"points":[{"id":hashlib.md5(url.encode()).hexdigest(),"vector":emb,"payload":{"url":url,"title":title,"excerpt":excerpt[:500]}}]})
|
|
except: pass
|
|
return {"title":title,"domain":domain,"category":category,"size":len(text)}
|
|
except: return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PUBLIC PAGES
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
def landing():
|
|
return HTMLResponse(LANDING_HTML)
|
|
|
|
@app.get("/pricing", response_class=HTMLResponse)
|
|
def pricing_page():
|
|
return HTMLResponse(PRICING_HTML)
|
|
|
|
@app.get("/signup", response_class=HTMLResponse)
|
|
def signup_page():
|
|
return HTMLResponse(SIGNUP_HTML)
|
|
|
|
@app.get("/dashboard", response_class=HTMLResponse)
|
|
def dashboard_page():
|
|
return HTMLResponse(DASHBOARD_HTML)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# AUTH API
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
@app.post("/api/signup")
|
|
async def api_signup(request: Request):
|
|
data = await request.json()
|
|
email = data.get("email","").strip().lower()
|
|
if not email or "@" not in email:
|
|
return JSONResponse({"error":"Valid email required"}, 400)
|
|
api_key = "sk-" + secrets.token_hex(24)
|
|
db = get_db()
|
|
cur = db.cursor()
|
|
try:
|
|
cur.execute("INSERT INTO users (email, password_hash, api_key, calls_remaining) VALUES (%s,%s,%s,0)", (email, "bcrypt_placeholder", api_key))
|
|
db.commit()
|
|
except psycopg2.errors.UniqueViolation:
|
|
db.rollback()
|
|
cur.execute("SELECT api_key FROM users WHERE email = %s", (email,))
|
|
api_key = cur.fetchone()[0]
|
|
db.close()
|
|
return {"email": email, "api_key": api_key, "message": "Signup successful. Purchase API calls at /pricing to start using the engine."}
|
|
|
|
@app.get("/api/my-usage")
|
|
async def my_usage(request: Request):
|
|
# Allow even with 0 calls — users need to see their balance
|
|
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
|
if not api_key:
|
|
raise HTTPException(401, "API key required.")
|
|
db = get_db()
|
|
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
|
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
|
user = cur.fetchone()
|
|
if not user:
|
|
db.close()
|
|
raise HTTPException(401, "Invalid API key.")
|
|
user = dict(user)
|
|
cur.execute("SELECT tool_name, COUNT(*) as cnt FROM usage_log WHERE user_id = %s GROUP BY tool_name ORDER BY cnt DESC", (user["id"],))
|
|
user["usage_by_tool"] = [dict(r) for r in cur.fetchall()]
|
|
db.close()
|
|
return user
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# BTCPAY INTEGRATION
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
@app.post("/api/create-invoice")
|
|
async def create_invoice(request: Request):
|
|
"""Create BTCPay invoice for $5 = 5 calls. Works even with 0 calls."""
|
|
api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
|
|
if not api_key:
|
|
raise HTTPException(401, "API key required.")
|
|
db = get_db()
|
|
cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
|
cur.execute("SELECT * FROM users WHERE api_key = %s", (api_key,))
|
|
user = cur.fetchone()
|
|
if not user:
|
|
db.close()
|
|
raise HTTPException(401, "Invalid API key.")
|
|
user = dict(user)
|
|
try:
|
|
r = httpx.post(f"{BTCPAY_URL}/api/v1/stores/{BTCPAY_STORE}/invoices",
|
|
headers={"Authorization": f"token {BTCPAY_KEY}", "Content-Type": "application/json"},
|
|
json={
|
|
"amount": str(PRICE_USD),
|
|
"currency": "USD",
|
|
"metadata": {
|
|
"user_id": user["id"],
|
|
"user_email": user["email"],
|
|
"calls": CALLS_PER_TIER,
|
|
"orderId": f"research-{user['id']}-{int(time.time())}"
|
|
},
|
|
"checkout": {"redirectURL": f"http://10.30.20.250:8000/dashboard"}
|
|
},
|
|
verify=False, timeout=15.0
|
|
)
|
|
inv = r.json()
|
|
# Log pending payment
|
|
db = get_db()
|
|
cur = db.cursor()
|
|
cur.execute("INSERT INTO payments (user_id, btcpay_invoice_id, amount_usd, calls_purchased, status) VALUES (%s,%s,%s,%s,'pending')",
|
|
(user["id"], inv["id"], PRICE_USD, CALLS_PER_TIER))
|
|
db.commit()
|
|
db.close()
|
|
return {"invoice_id": inv["id"], "checkout_url": inv["checkoutLink"], "amount": f"${PRICE_USD}", "calls": CALLS_PER_TIER}
|
|
except Exception as e:
|
|
return JSONResponse({"error": str(e)}, 500)
|
|
|
|
@app.post("/webhook/btcpay")
|
|
async def btcpay_webhook(request: Request):
|
|
"""BTCPay calls this when invoice is settled."""
|
|
body = await request.json()
|
|
event_type = body.get("type", "")
|
|
invoice_id = body.get("invoiceId", "")
|
|
metadata = body.get("metadata", {})
|
|
|
|
# Handle nested metadata (BTCPay wraps it)
|
|
if isinstance(metadata, dict):
|
|
user_id = metadata.get("user_id")
|
|
calls = metadata.get("calls", CALLS_PER_TIER)
|
|
else:
|
|
user_id = None
|
|
calls = CALLS_PER_TIER
|
|
|
|
if event_type in ("InvoiceSettled", "InvoiceProcessing") and user_id:
|
|
db = get_db()
|
|
cur = db.cursor()
|
|
cur.execute("UPDATE payments SET status = 'settled', settled_at = NOW() WHERE btcpay_invoice_id = %s", (invoice_id,))
|
|
cur.execute("UPDATE users SET calls_remaining = calls_remaining + %s WHERE id = %s", (calls, user_id))
|
|
db.commit()
|
|
db.close()
|
|
|
|
return {"status": "ok"}
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# API TOOLS (auth-gated)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
@app.get("/api/status")
|
|
def status():
|
|
svc = {}
|
|
try:
|
|
r = client.get(f"{OPENSEARCH_URL}/_cluster/health")
|
|
cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json()
|
|
svc["opensearch"] = {"status": r.json().get("status"), "documents": cnt.get("count",0)}
|
|
except: svc["opensearch"] = {"status":"down"}
|
|
try:
|
|
client.get(f"{QDRANT_URL}/healthz")
|
|
col = client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}").json()
|
|
svc["qdrant"] = {"status":"ok","points":col.get("result",{}).get("points_count",0)}
|
|
except: svc["qdrant"] = {"status":"down"}
|
|
try:
|
|
client.get(f"{YACY_URL}/yacysearch.json",params={"query":"test","maximumRecords":1})
|
|
svc["yacy"] = {"status":"ok"}
|
|
except: svc["yacy"] = {"status":"down"}
|
|
try:
|
|
r = ollama.get("/api/tags")
|
|
svc["ollama"] = {"status":"ok","models":len(r.json().get("models",[]))}
|
|
except: svc["ollama"] = {"status":"down"}
|
|
# DB stats
|
|
try:
|
|
db = get_db()
|
|
cur = db.cursor()
|
|
cur.execute("SELECT COUNT(*) FROM users")
|
|
users = cur.fetchone()[0]
|
|
cur.execute("SELECT COALESCE(SUM(total_calls),0) FROM users")
|
|
total = cur.fetchone()[0]
|
|
cur.execute("SELECT COUNT(*) FROM payments WHERE status = 'settled'")
|
|
payments = cur.fetchone()[0]
|
|
db.close()
|
|
svc["business"] = {"status":"ok","users":users,"total_api_calls":total,"payments_settled":payments}
|
|
except: svc["business"] = {"status":"down"}
|
|
return {"services":svc}
|
|
|
|
@app.get("/api/search")
|
|
def search_web(request: Request, q: str = Query(...), category: str = "", limit: int = 10):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "search_web", request)
|
|
_ensure_index()
|
|
body = {"size":limit,"query":{"bool":{"must":[{"multi_match":{"query":q,"fields":["title^3","content","excerpt"]}}]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":2}}}}
|
|
if category: body["query"]["bool"]["filter"] = [{"term":{"category":category}}]
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json=body)
|
|
result = r.json()
|
|
hits = []
|
|
for h in result.get("hits",{}).get("hits",[]):
|
|
src = h["_source"]
|
|
hits.append({"url":src.get("url"),"title":src.get("title"),"excerpt":src.get("excerpt") or (h.get("highlight",{}).get("content",[""])[0]),"category":src.get("category"),"crawled_at":src.get("crawled_at"),"score":h["_score"]})
|
|
return {"query":q,"total":result.get("hits",{}).get("total",{}).get("value",0),"hits":hits}
|
|
except Exception as e:
|
|
return {"query":q,"total":0,"hits":[],"note":str(e)}
|
|
|
|
@app.get("/api/semantic-search")
|
|
def semantic_search(request: Request, q: str = Query(...), limit: int = 10):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "semantic_search", request)
|
|
emb = _get_embedding(q)
|
|
if not emb: return {"hits":[],"error":"Embedding model unavailable"}
|
|
try:
|
|
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":limit,"with_payload":True,"with_vector":False})
|
|
hits = [{"url":p.get("payload",{}).get("url"),"title":p.get("payload",{}).get("title"),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:300],"score":p.get("score")} for p in r.json().get("result",[])]
|
|
return {"query":q,"total":len(hits),"hits":hits}
|
|
except Exception as e:
|
|
return {"hits":[],"error":str(e)}
|
|
|
|
@app.get("/api/crawl")
|
|
def crawl_url(request: Request, url: str = Query(...), depth: int = 1):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "crawl_url", request)
|
|
indexed = _fetch_and_index(url)
|
|
try:
|
|
client.get(f"{YACY_URL}/Crawler_p.json", params={"crawlingDomMaxPages":50,"crawlingDepth":depth,"crawlingStart":url,"crawlingQ":"on","bookmarkTitle":"research","bookmarkFolder":"/research","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"})
|
|
return {"status":"crawl_started","url":url,"depth":depth,"indexed":indexed}
|
|
except:
|
|
return {"status":"indexed_only","url":url,"indexed":indexed}
|
|
|
|
@app.get("/api/research")
|
|
def research_topic(request: Request, topic: str = Query(...)):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "research_topic", request)
|
|
steps = []; kw_result = {"hits":[]}; sem_result = {"hits":[]}
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":5,"query":{"multi_match":{"query":topic,"fields":["title^3","content","excerpt"]}},"highlight":{"fields":{"content":{"fragment_size":200,"number_of_fragments":1}}}})
|
|
hits = [{"title":h["_source"].get("title"),"excerpt":h.get("highlight",{}).get("content",[h["_source"].get("excerpt","")])[0]} for h in r.json().get("hits",{}).get("hits",[])]
|
|
kw_result = {"total":len(hits),"hits":hits}; steps.append("keyword_search_done")
|
|
except: steps.append("keyword_search_skipped")
|
|
try:
|
|
emb = _get_embedding(topic)
|
|
if emb:
|
|
r = client.post(f"{QDRANT_URL}/collections/{INDEX_NAME}/points/search",json={"vector":emb,"limit":5,"with_payload":True})
|
|
hits2 = [{"title":p.get("payload",{}).get("title",""),"excerpt":str(p.get("payload",{}).get("excerpt",""))[:200],"score":p.get("score")} for p in r.json().get("result",[])]
|
|
sem_result = {"total":len(hits2),"hits":hits2}; steps.append("semantic_search_done")
|
|
except: steps.append("semantic_search_skipped")
|
|
try:
|
|
r = client.get(f"{YACY_URL}/yacysearch.json",params={"query":topic,"maximumRecords":10,"resource":"global"})
|
|
discovered = [item.get("link") for ch in r.json().get("channels",[]) for item in ch.get("items",[]) if item.get("link")]
|
|
for url in discovered[:10]:
|
|
try: client.get(f"{YACY_URL}/Crawler_p.json",params={"crawlingDomMaxPages":10,"crawlingDepth":0,"crawlingStart":url,"crawlingQ":"on","indexText":"on","indexMedia":"on","crawlingMode":"url","cachePolicy":"iffresh"},timeout=5.0)
|
|
except: pass
|
|
steps.append(f"crawl_dispatched_{len(discovered[:10])}")
|
|
except: steps.append("crawl_skipped")
|
|
all_src = ""
|
|
for h in kw_result.get("hits",[])[:3] + sem_result.get("hits",[])[:3]:
|
|
all_src += f"- {h.get('title','Unknown')}: {h.get('excerpt','')[:200]}\n"
|
|
summary = _ai_chat(f"Research topic: {topic}\n\nSources:\n{all_src}\n\nConcise research summary (3-5 paragraphs): key findings, important sources, knowledge gaps, next steps.") if all_src else ""
|
|
return {"topic":topic,"steps":steps,"keyword_results":kw_result,"semantic_results":sem_result,"ai_summary":summary}
|
|
|
|
@app.get("/api/report")
|
|
def create_report(request: Request, topic: str = Query(...), sources: str = ""):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "create_report", request)
|
|
if sources:
|
|
urls = [u.strip() for u in sources.split(",") if u.strip()]
|
|
else:
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":8,"query":{"multi_match":{"query":topic,"fields":["title^3","content"]}}})
|
|
urls = [h["_source"]["url"] for h in r.json().get("hits",{}).get("hits",[])]
|
|
except: urls = []
|
|
gathered = ""
|
|
for url in urls[:8]:
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
|
hits = r.json().get("hits",{}).get("hits",[])
|
|
if hits:
|
|
src = hits[0]["_source"]
|
|
gathered += f"\n\n### {src.get('title','Source')}\nURL: {url}\n{src.get('content',src.get('excerpt',''))[:1500]}"
|
|
except: pass
|
|
report = _ai_chat(f"Research topic: {topic}\nSources:{gathered if gathered else ' No sources found.'}\n\nGenerate a comprehensive report:\n1. Executive Summary\n2. Key Findings (numbered)\n3. Source Analysis\n4. Knowledge Gaps\n5. Recommendations\n\nBe thorough, use markdown, cite sources.", system="You are a senior research analyst. Produce thorough, structured reports.")
|
|
return {"topic":topic,"sources_used":len(urls),"report":report}
|
|
|
|
@app.get("/api/document")
|
|
def retrieve_document(request: Request, url: str = Query(...)):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "retrieve_document", request)
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
|
hits = r.json().get("hits",{}).get("hits",[])
|
|
if not hits: return {"error":"Not found","url":url}
|
|
src = hits[0]["_source"]
|
|
return {"url":src.get("url"),"title":src.get("title"),"content":src.get("content","")[:10000],"excerpt":src.get("excerpt"),"category":src.get("category"),"crawled_at":src.get("crawled_at")}
|
|
except Exception as e:
|
|
return {"error":str(e),"url":url}
|
|
|
|
@app.get("/api/summarize")
|
|
def summarize_sources(request: Request, urls: str = Query(...), instruction: str = "Summarize key points"):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "summarize_sources", request)
|
|
url_list = [u.strip() for u in urls.split(",") if u.strip()]
|
|
combined = ""
|
|
for url in url_list[:5]:
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
|
hits = r.json().get("hits",{}).get("hits",[])
|
|
if hits:
|
|
src = hits[0]["_source"]
|
|
combined += f"\n\n--- {url} ---\n{src.get('title','')}\n{src.get('content',src.get('excerpt',''))[:2000]}"
|
|
except: pass
|
|
if not combined.strip(): return {"error":"No content retrieved"}
|
|
summary = _ai_chat(f"Instruction: {instruction}\n\nSources:{combined}\n\nProvide a structured summary.")
|
|
return {"instruction":instruction,"sources":len(url_list),"summary":summary}
|
|
|
|
@app.get("/api/extract")
|
|
def extract_information(request: Request, url: str = Query(...), schema: str = Query("company names, products, prices")):
|
|
user = _auth(request)
|
|
_track_usage(user["id"], "extract_information", request)
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={"size":1,"query":{"term":{"url":url}}})
|
|
hits = r.json().get("hits",{}).get("hits",[])
|
|
if not hits: return {"error":"Not found","url":url}
|
|
content = hits[0]["_source"].get("content","")[:8000]
|
|
result = _ai_chat(f"Extract: {schema}\n\nDocument:\n{content}\n\nReturn ONLY valid JSON.", system="Extract structured data. Return ONLY valid JSON.")
|
|
return {"url":url,"schema":schema,"extracted":result}
|
|
except Exception as e:
|
|
return {"error":str(e)}
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# UI TEMPLATES
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
CSS = """
|
|
:root{--bg:#050508;--surface:#0c0c14;--border:#1e1e30;--accent:#7c3aed;--accent2:#a855f7;--gold:#f59e0b;--text:#f4f4f5;--muted:#a1a1aa;--green:#22c55e;--red:#ef4444}
|
|
*{box-sizing:border-box;margin:0;padding:0}
|
|
body{font-family:'Inter',system-ui,sans-serif;background:var(--bg);color:var(--text);min-height:100vh;overflow-x:hidden}
|
|
.gradient-bg{background:radial-gradient(ellipse 80% 50% at 50% -20%,rgba(120,60,255,0.15),transparent)}
|
|
.header{background:rgba(12,12,20,0.8);backdrop-filter:blur(20px);border-bottom:1px solid var(--border);padding:16px 32px;display:flex;justify-content:space-between;align-items:center;position:sticky;top:0;z-index:100}
|
|
.header h1{font-size:1.3rem;font-weight:800;background:linear-gradient(135deg,var(--accent),var(--accent2),#ec4899);-webkit-background-clip:text;-webkit-text-fill-color:transparent}
|
|
.header nav{display:flex;gap:20px;align-items:center}
|
|
.header nav a{color:var(--muted);text-decoration:none;font-size:0.9rem;transition:color .2s}
|
|
.header nav a:hover{color:var(--text)}
|
|
.btn{padding:12px 24px;border-radius:12px;border:none;font-weight:600;font-size:0.95rem;cursor:pointer;transition:all .2s;text-decoration:none;display:inline-block}
|
|
.btn-primary{background:var(--accent);color:white}
|
|
.btn-primary:hover{background:var(--accent2);transform:translateY(-1px);box-shadow:0 8px 25px rgba(124,58,237,0.3)}
|
|
.btn-gold{background:linear-gradient(135deg,var(--gold),#fbbf24);color:#1a1a1a}
|
|
.btn-gold:hover{transform:translateY(-1px);box-shadow:0 8px 25px rgba(245,158,11,0.3)}
|
|
.btn-outline{background:transparent;border:1px solid var(--border);color:var(--text)}
|
|
.container{max-width:1200px;margin:0 auto;padding:40px 24px}
|
|
.hero{text-align:center;padding:80px 0 60px}
|
|
.hero h2{font-size:3rem;font-weight:900;letter-spacing:-0.04em;line-height:1.1;margin-bottom:20px;background:linear-gradient(135deg,var(--text),var(--accent2));-webkit-background-clip:text;-webkit-text-fill-color:transparent}
|
|
.hero p{color:var(--muted);font-size:1.2rem;max-width:650px;margin:0 auto 32px;line-height:1.6}
|
|
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:20px;margin:40px 0}
|
|
.card{background:var(--surface);border:1px solid var(--border);border-radius:16px;padding:28px;transition:all .2s}
|
|
.card:hover{border-color:var(--accent);transform:translateY(-2px)}
|
|
.card h3{font-size:1.2rem;margin-bottom:10px}
|
|
.card p{color:var(--muted);font-size:0.9rem;line-height:1.5}
|
|
.price{font-size:3rem;font-weight:900;text-align:center;margin:20px 0}
|
|
.price span{font-size:1rem;color:var(--muted);font-weight:400}
|
|
.feature-list{list-style:none;margin:20px 0}
|
|
.feature-list li{padding:8px 0;color:var(--muted);font-size:0.9rem}
|
|
.feature-list li::before{content:'✓ ';color:var(--green);font-weight:700}
|
|
pre{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:16px;overflow-x:auto;font-size:0.85rem;margin:10px 0}
|
|
.footer{text-align:center;padding:40px;color:var(--muted);font-size:0.85rem;border-top:1px solid var(--border);margin-top:60px}
|
|
.footer a{color:var(--accent2);text-decoration:none}
|
|
input,textarea{width:100%;padding:14px 18px;border-radius:12px;border:1px solid var(--border);background:var(--bg);color:var(--text);font-size:1rem;outline:none;transition:border-color .2s;margin-bottom:12px}
|
|
input:focus,textarea:focus{border-color:var(--accent)}
|
|
"""
|
|
|
|
LANDING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>AI Research Engine — Private Knowledge Cloud</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}.glow{{position:absolute;width:600px;height:600px;border-radius:50%;filter:blur(120px);opacity:0.12;pointer-events:none}}.glow-1{{background:var(--accent);top:-200px;left:-100px}}.glow-2{{background:#ec4899;bottom:-200px;right:-100px}}</style></head><body class="gradient-bg"><div class="glow glow-1"></div><div class="glow glow-2"></div><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>The Ultimate Source of<br>Uncensored Internet Knowledge</h2><p>A self-hosted AI workforce that continuously gathers, organizes, and acts on information. Your private research cloud — no filters, no tracking, no limits.</p><div style="display:flex;gap:16px;justify-content:center;flex-wrap:wrap"><a href="/signup" class="btn btn-primary" style="font-size:1.1rem;padding:16px 36px">Get Started — Free Signup</a><a href="/pricing" class="btn btn-gold" style="font-size:1.1rem;padding:16px 36px">$5 Bitcoin = 5 API Calls ⚡</a></div></div><div class="grid"><div class="card"><h3>🔍 Deep Search</h3><p>Full-text search across millions of indexed documents. Find anything — no Google bubble, no censorship.</p></div><div class="card"><h3>🧬 Semantic Understanding</h3><p>Search by meaning, not keywords. Our vector engine finds conceptually related content even when the words differ.</p></div><div class="card"><h3>🕷️ Autonomous Crawling</h3><p>Point it at a topic and it discovers, crawls, and indexes every relevant source. Never miss a thing.</p></div><div class="card"><h3>🤖 AI Synthesis</h3><p>Local LLMs summarize, extract, and report. Your research, analyzed by AI running on our hardware.</p></div><div class="card"><h3>🌍 Geo-Distributed</h3><p>Crawls route through proxies in Tokyo, London, and Sydney. Bypass regional blocks automatically.</p></div><div class="card"><h3>🔐 Private & Self-Hosted</h3><p>Everything runs on our metal. No third-party APIs, no data harvesting, no surveillance.</p></div></div><div style="text-align:center;padding:40px 0"><h3 style="font-size:1.5rem;margin-bottom:16px">Trusted by AI Agents Worldwide</h3><p style="color:var(--muted);max-width:600px;margin:0 auto">10 MCP tools. One API key. Infinite knowledge. Agents connect and research autonomously — no browser needed.</p></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p><p style="margin-top:8px">AI Research Engine v2.0 · Self-hosted · No KYC · Bitcoin Only</p></div></div></body></html>"""
|
|
|
|
PRICING_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Pricing — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800;900&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Simple Bitcoin Pricing</h2><p>No subscriptions. No KYC. Pay with Bitcoin, get API calls.</p></div><div style="max-width:500px;margin:0 auto"><div class="card" style="text-align:center;border-color:var(--accent);border-width:2px"><h3>🚀 Researcher Tier</h3><div class="price">$5<span> USD</span></div><p style="color:var(--muted);margin-bottom:16px">in Bitcoin · Lightning ⚡</p><ul class="feature-list" style="text-align:left;max-width:300px;margin:0 auto"><li>5 API calls</li><li>Full access to all 10 tools</li><li>Search, crawl, research, report</li><li>No expiration</li><li>No KYC required</li></ul><div style="margin-top:24px"><p style="color:var(--muted);font-size:0.85rem;margin-bottom:12px">👇 Sign up first, then buy from your dashboard</p><a href="/signup" class="btn btn-gold" style="width:100%">Sign Up & Get Your API Key</a></div></div></div><div class="grid" style="max-width:800px;margin:40px auto"><div class="card"><h3>⚡ Lightning Fast</h3><p>Payments settle in seconds. Your API calls are credited instantly.</p></div><div class="card"><h3>🔑 Bring Your Own Key</h3><p>Use your API key in any MCP client, script, or AI agent.</p></div><div class="card"><h3>📊 Track Usage</h3><p>Real-time dashboard shows your remaining calls and history.</p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a> · <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">☕ Buy Me a Coffee</a></p></div></div></body></html>"""
|
|
|
|
SIGNUP_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Sign Up — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero"><h2>Get Your API Key</h2><p>No KYC. Just an email. Your key is generated instantly.</p></div><div class="card" style="max-width:500px;margin:0 auto"><input type="email" id="email" placeholder="you@email.com" style="font-size:1.1rem"><button class="btn btn-primary" onclick="signup()" style="width:100%;font-size:1.1rem">Generate My API Key</button><div id="result" style="margin-top:20px;display:none"><p style="color:var(--green);margin-bottom:8px">✅ Your API key is ready:</p><pre id="apikey" style="cursor:pointer;word-break:break-all" onclick="copyKey()"></pre><p style="color:var(--muted);font-size:0.85rem;margin-top:8px">Click key to copy. Store it safely. No password reset — it's yours.</p><p style="color:var(--muted);font-size:0.85rem">Next: <a href="/pricing" style="color:var(--accent2)">buy API calls</a> or <a href="/dashboard" style="color:var(--accent2)">go to dashboard</a></p></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
|
|
async function signup(){{const e=document.getElementById('email').value;if(!e)return;const r=await fetch('/api/signup',{{method:'POST',headers:{{'Content-Type':'application/json'}},body:JSON.stringify({{email:e}})}});const d=await r.json();document.getElementById('result').style.display='block';document.getElementById('apikey').textContent=d.api_key||d.error}}
|
|
function copyKey(){{const t=document.getElementById('apikey').textContent;navigator.clipboard.writeText(t);alert('API key copied!')}}
|
|
</script></body></html>"""
|
|
|
|
DASHBOARD_HTML = f"""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>Dashboard — AI Research Engine</title><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700;800&display=swap" rel="stylesheet"><style>{CSS}</style></head><body class="gradient-bg"><div class="header"><h1>🧠 AI Research Engine</h1><nav><a href="/">Home</a><a href="/pricing">Pricing</a><a href="/signup">Sign Up</a><a href="/dashboard">Dashboard</a></nav></div><div class="container"><div class="hero" style="padding:40px 0"><h2>Dashboard</h2><p>Your account, usage, and API access.</p></div><div id="login-section"><div class="card" style="max-width:500px;margin:0 auto"><h3>🔑 Enter Your API Key</h3><input type="text" id="apikey-input" placeholder="sk-..." style="font-family:monospace"><button class="btn btn-primary" onclick="login()" style="width:100%">View My Dashboard</button></div></div><div id="dashboard-section" style="display:none"><div class="grid" id="stats"></div><div class="card" style="margin-bottom:20px"><h3>⚡ Buy API Calls</h3><p style="color:var(--muted);margin-bottom:16px">$5 Bitcoin = 5 API calls. Pay with Lightning.</p><button class="btn btn-gold" onclick="buyCalls()">Pay $5 with Bitcoin ⚡</button><div id="invoice-result" style="margin-top:16px"></div></div><div class="card"><h3>📋 Quick Reference</h3><pre style="font-size:0.8rem"># Use your key in any HTTP request:
|
|
curl -H "X-API-Key: YOUR_KEY" \\
|
|
"http://10.30.20.249:8000/api/search?q=your+query"
|
|
|
|
# Or as a query parameter:
|
|
curl "http://10.30.20.249:8000/api/search?q=test&api_key=YOUR_KEY"</pre></div></div><div class="footer"><p>Created by <a href="https://buymeacoffee.com/r26xrthzttg" target="_blank">drjones</a></p></div></div><script>
|
|
let apiKey='';
|
|
async function login(){{apiKey=document.getElementById('apikey-input').value;if(!apiKey)return;loadDashboard()}}
|
|
async function loadDashboard(){{try{{const r=await fetch('/api/my-usage?api_key='+apiKey);const d=await r.json();document.getElementById('login-section').style.display='none';document.getElementById('dashboard-section').style.display='block';let h='<div class="card"><h3>📧 '+d.email+'</h3><div class="price" style="font-size:2rem">'+d.calls_remaining+'<span> calls left</span></div><p style="color:var(--muted)">'+d.total_calls+' total · since '+d.created_at+'</p></div>';h+='<div class="card"><h3>🔑 API Key</h3><pre style="word-break:break-all;cursor:pointer" onclick="navigator.clipboard.writeText(\''+d.api_key+'\')">'+d.api_key+'</pre><p style="color:var(--muted);font-size:0.8rem">Click to copy</p></div>';if(d.usage_by_tool){{h+='<div class="card"><h3>📊 Usage by Tool</h3>';for(const t of d.usage_by_tool)h+='<p style="display:flex;justify-content:space-between"><span>'+t.tool_name+'</span><span style="color:var(--accent2)">'+t.cnt+' calls</span></p>';h+='</div>'}}if(d.is_admin)h+='<div class="card" style="border-color:var(--gold)"><h3>👑 Admin Account</h3><p style="color:var(--gold)">Unlimited calls · Full system access</p></div>';document.getElementById('stats').innerHTML=h}}catch(e){{alert('Invalid API key')}}}}
|
|
async function buyCalls(){{document.getElementById('invoice-result').innerHTML='<p style="color:var(--gold)">Creating invoice...</p>';const r=await fetch('/api/create-invoice?api_key='+apiKey,{{method:'POST'}});const d=await r.json();if(d.checkout_url){{document.getElementById('invoice-result').innerHTML='<a href="'+d.checkout_url+'" target="_blank" class="btn btn-gold" style="width:100%">Pay '+d.amount+' with Bitcoin →</a><p style="color:var(--muted);margin-top:8px;font-size:0.85rem">Opens BTCPay checkout. Refresh after payment.</p>'}}else{{document.getElementById('invoice-result').innerHTML='<p style="color:var(--red)">Error: '+JSON.stringify(d)+'</p>'}}}}
|
|
</script></body></html>"""
|
|
|
|
|
|
# ── Startup ──────────────────────────────────────────────────
|
|
|
|
if __name__ == "__main__":
|
|
uvicorn.run(app, host="0.0.0.0", port=8000)
|