34061a4ac1
- 10 MCP tools via thin proxy on MacBook - Backend REST API + Dashboard on CT 145 Docker - Services: YaCy crawler, OpenSearch index, Qdrant vectors, Ollama LLM - All 4 services healthy and verified
227 lines
11 KiB
Python
227 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Research Engine Dashboard — FastAPI web UI
|
|
Runs on CT 145 port 8000
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
import httpx
|
|
from fastapi import FastAPI, Request, Query
|
|
from fastapi.responses import HTMLResponse, JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(os.path.expanduser("~/ai-research-engine/.env"))
|
|
|
|
app = FastAPI(title="AI Research Engine")
|
|
|
|
OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://localhost:9200")
|
|
QDRANT_URL = os.getenv("QDRANT_URL", "http://10.30.20.68:6333")
|
|
YACY_URL = os.getenv("YACY_URL", "http://localhost:8090")
|
|
INDEX_NAME = "research_docs"
|
|
|
|
client = httpx.Client(timeout=10.0)
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def dashboard():
|
|
return HTMLResponse("""
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>AI Research Engine</title>
|
|
<style>
|
|
:root { --bg: #0f0f0f; --card: #1a1a1a; --accent: #6366f1; --text: #e2e8f0; --muted: #94a3b8; --green: #22c55e; --red: #ef4444; }
|
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body { font-family: 'Inter', -apple-system, sans-serif; background: var(--bg); color: var(--text); min-height: 100vh; }
|
|
.header { background: var(--card); border-bottom: 1px solid #262626; padding: 20px 40px; display: flex; justify-content: space-between; align-items: center; }
|
|
.header h1 { font-size: 1.5rem; font-weight: 700; background: linear-gradient(135deg, var(--accent), #a855f7); -webkit-background-clip: text; -webkit-text-fill-color: transparent; }
|
|
.container { max-width: 1200px; margin: 0 auto; padding: 30px 20px; }
|
|
.search-box { display: flex; gap: 10px; margin-bottom: 30px; }
|
|
.search-box input { flex: 1; padding: 14px 20px; border-radius: 12px; border: 1px solid #333; background: var(--card); color: var(--text); font-size: 1rem; }
|
|
.search-box button { padding: 14px 28px; border-radius: 12px; border: none; background: var(--accent); color: white; font-weight: 600; cursor: pointer; font-size: 1rem; }
|
|
.search-box button:hover { opacity: 0.9; }
|
|
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; margin-bottom: 30px; }
|
|
.stat-card { background: var(--card); border: 1px solid #262626; border-radius: 12px; padding: 20px; }
|
|
.stat-card .label { color: var(--muted); font-size: 0.85rem; margin-bottom: 4px; }
|
|
.stat-card .value { font-size: 1.8rem; font-weight: 700; }
|
|
.stat-card .ok { color: var(--green); } .stat-card .down { color: var(--red); }
|
|
.results { display: flex; flex-direction: column; gap: 12px; }
|
|
.result-card { background: var(--card); border: 1px solid #262626; border-radius: 12px; padding: 20px; }
|
|
.result-card h3 { margin-bottom: 6px; }
|
|
.result-card h3 a { color: var(--accent); text-decoration: none; }
|
|
.result-card .excerpt { color: var(--muted); font-size: 0.9rem; line-height: 1.5; }
|
|
.result-card .meta { color: #64748b; font-size: 0.8rem; margin-top: 8px; }
|
|
.action-bar { display: flex; gap: 8px; margin-bottom: 20px; flex-wrap: wrap; }
|
|
.action-bar button { padding: 10px 20px; border-radius: 8px; border: 1px solid #333; background: var(--card); color: var(--text); cursor: pointer; font-size: 0.9rem; }
|
|
.action-bar button:hover { border-color: var(--accent); }
|
|
.loading { text-align: center; padding: 40px; color: var(--muted); }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h1>🧠 AI Research Engine</h1>
|
|
<span style="color:var(--muted);font-size:0.85rem">Self-hosted knowledge acquisition</span>
|
|
</div>
|
|
<div class="container">
|
|
<div class="search-box">
|
|
<input type="text" id="query" placeholder="Research anything..." onkeydown="if(event.key==='Enter')search()">
|
|
<button onclick="search()">Search</button>
|
|
<button onclick="research()" style="background:#a855f7;">Deep Research</button>
|
|
</div>
|
|
<div class="action-bar">
|
|
<button onclick="loadStatus()">📊 System Status</button>
|
|
<button onclick="document.getElementById('crawl-url').style.display='block'">🕷️ Crawl URL</button>
|
|
</div>
|
|
<div id="crawl-url" style="display:none;margin-bottom:20px;display:none;">
|
|
<input type="text" id="crawl-input" placeholder="https://example.com" style="padding:10px;width:400px;border-radius:8px;border:1px solid #333;background:var(--card);color:var(--text);">
|
|
<button onclick="crawl()" style="padding:10px 20px;border-radius:8px;border:none;background:var(--accent);color:white;cursor:pointer;margin-left:8px;">Crawl</button>
|
|
</div>
|
|
<div class="grid" id="stats"></div>
|
|
<div id="output"></div>
|
|
</div>
|
|
<script>
|
|
async function api(path) { const r = await fetch(path); return r.json(); }
|
|
async function loadStatus() {
|
|
try { const d = await api('/api/status'); renderStatus(d); } catch(e) { document.getElementById('output').innerHTML='<div class="loading">Backend unreachable — is docker-compose running?</div>'; }
|
|
}
|
|
async function search() {
|
|
const q = document.getElementById('query').value;
|
|
if(!q) return;
|
|
document.getElementById('output').innerHTML='<div class="loading">Searching...</div>';
|
|
try { const d = await api('/api/search?q='+encodeURIComponent(q)); renderResults(d); } catch(e) { document.getElementById('output').innerHTML='<div class="loading">Error: '+e.message+'</div>'; }
|
|
}
|
|
async function research() {
|
|
const q = document.getElementById('query').value;
|
|
if(!q) return;
|
|
document.getElementById('output').innerHTML='<div class="loading">Researching (this may take a minute)...</div>';
|
|
try { const d = await api('/api/research?topic='+encodeURIComponent(q)); renderResearch(d); } catch(e) { document.getElementById('output').innerHTML='<div class="loading">Error: '+e.message+'</div>'; }
|
|
}
|
|
async function crawl() {
|
|
const url = document.getElementById('crawl-input').value;
|
|
if(!url) return;
|
|
document.getElementById('output').innerHTML='<div class="loading">Crawling...</div>';
|
|
try { const d = await api('/api/crawl?url='+encodeURIComponent(url)); document.getElementById('output').innerHTML='<pre style="background:var(--card);padding:16px;border-radius:8px;overflow-x:auto;">'+JSON.stringify(d,null,2)+'</pre>'; } catch(e) {}
|
|
}
|
|
function renderStatus(d) {
|
|
let html = '';
|
|
for(const [name, info] of Object.entries(d.services||{})) {
|
|
const cls = info.status==='ok'||info.status==='green'?'ok':'down';
|
|
html += `<div class="stat-card"><div class="label">${name}</div><div class="value ${cls}">${info.status||'down'}</div><div style="color:var(--muted);font-size:0.8rem;margin-top:4px;">${info.documents!==undefined?'📄 '+info.documents+' docs':''}${info.points!==undefined?'🔢 '+info.points+' vectors':''}</div></div>`;
|
|
}
|
|
document.getElementById('stats').innerHTML = html;
|
|
}
|
|
function renderResults(d) {
|
|
let html = `<p style="color:var(--muted);margin-bottom:16px;">Found ${d.total||0} results for "${d.query||''}"</p><div class="results">`;
|
|
for(const h of (d.hits||[])) {
|
|
html += `<div class="result-card"><h3><a href="${h.url||'#'}" target="_blank">${h.title||'Untitled'}</a></h3><div class="excerpt">${h.excerpt||''}</div><div class="meta">Score: ${(h.score||0).toFixed(2)} · ${h.category||''} · ${h.crawled_at||''}</div></div>`;
|
|
}
|
|
html += '</div>';
|
|
document.getElementById('output').innerHTML = html;
|
|
}
|
|
function renderResearch(d) {
|
|
let html = `<div class="result-card"><h3>Research: ${d.topic||''}</h3>`;
|
|
if(d.ai_summary) html += `<div class="excerpt" style="white-space:pre-wrap;">${d.ai_summary}</div>`;
|
|
html += `<div class="meta">Steps: ${(d.steps||[]).join(', ')}</div></div>`;
|
|
document.getElementById('output').innerHTML = html;
|
|
}
|
|
loadStatus();
|
|
</script>
|
|
</body>
|
|
</html>
|
|
""")
|
|
|
|
@app.get("/api/status")
|
|
async def status():
|
|
svc = {}
|
|
# OpenSearch
|
|
try:
|
|
r = client.get(f"{OPENSEARCH_URL}/_cluster/health")
|
|
h = r.json()
|
|
cnt = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_count").json()
|
|
svc["opensearch"] = {"status": h.get("status"), "documents": cnt.get("count", 0)}
|
|
except Exception as e:
|
|
svc["opensearch"] = {"status": "down", "error": str(e)}
|
|
# Qdrant
|
|
try:
|
|
r = client.get(f"{QDRANT_URL}/health")
|
|
if r.status_code == 200:
|
|
col = client.get(f"{QDRANT_URL}/collections/{INDEX_NAME}").json()
|
|
svc["qdrant"] = {"status": "ok", "points": col.get("result", {}).get("points_count", 0)}
|
|
except Exception as e:
|
|
svc["qdrant"] = {"status": "down", "error": str(e)}
|
|
# YaCy
|
|
try:
|
|
r = client.get(f"{YACY_URL}/api/status.json")
|
|
svc["yacy"] = {"status": r.json().get("status", "unknown")}
|
|
except Exception as e:
|
|
svc["yacy"] = {"status": "down", "error": str(e)}
|
|
# Ollama
|
|
try:
|
|
r = httpx.get("http://10.30.20.186:11434/api/tags", timeout=5.0)
|
|
svc["ollama"] = {"status": "ok", "models": len(r.json().get("models", []))}
|
|
except Exception as e:
|
|
svc["ollama"] = {"status": "down", "error": str(e)}
|
|
|
|
return {"services": svc}
|
|
|
|
@app.get("/api/search")
|
|
async def search(q: str = Query(...), limit: int = 10):
|
|
try:
|
|
r = client.post(f"{OPENSEARCH_URL}/{INDEX_NAME}/_search", json={
|
|
"size": limit,
|
|
"query": {"multi_match": {"query": q, "fields": ["title^3", "content", "excerpt"]}},
|
|
"highlight": {"fields": {"content": {"fragment_size": 200, "number_of_fragments": 2}}},
|
|
})
|
|
hits = []
|
|
for h in r.json().get("hits", {}).get("hits", []):
|
|
src = h["_source"]
|
|
hits.append({
|
|
"url": src.get("url"), "title": src.get("title"),
|
|
"excerpt": h.get("highlight", {}).get("content", [src.get("excerpt", "")])[0],
|
|
"category": src.get("category"), "crawled_at": src.get("crawled_at"),
|
|
"score": h["_score"],
|
|
})
|
|
return {"query": q, "total": r.json().get("hits", {}).get("total", {}).get("value", 0), "hits": hits}
|
|
except Exception as e:
|
|
return {"query": q, "total": 0, "hits": [], "error": str(e)}
|
|
|
|
@app.get("/api/crawl")
|
|
async def crawl(url: str = Query(...)):
|
|
try:
|
|
r = client.get(f"{YACY_URL}/Crawler_p.json", params={
|
|
"crawlingDomMaxPages": 50, "crawlingDepth": 1,
|
|
"crawlingStart": url, "crawlingQ": "on",
|
|
"bookmarkTitle": "research", "bookmarkFolder": "/research",
|
|
"indexText": "on", "indexMedia": "on",
|
|
"crawlingMode": "url", "cachePolicy": "iffresh",
|
|
})
|
|
return {"status": "crawl_started", "url": url, "yacy": r.json()}
|
|
except Exception as e:
|
|
return {"status": "error", "url": url, "error": str(e)}
|
|
|
|
@app.get("/api/research")
|
|
async def research(topic: str = Query(...)):
|
|
# Proxy to MCP server's research_topic via command
|
|
import subprocess, sys
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", f"""
|
|
import json, sys
|
|
sys.path.insert(0, '/Users/drjones/ai-research-engine')
|
|
from server import research_topic
|
|
print(research_topic("{topic}"))
|
|
"""],
|
|
capture_output=True, text=True, timeout=120,
|
|
env={**os.environ, "PYTHONPATH": "/Users/drjones/ai-research-engine"}
|
|
)
|
|
try:
|
|
return json.loads(result.stdout)
|
|
except:
|
|
return {"error": result.stderr, "topic": topic}
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8000)
|