import express, { Request, Response, NextFunction } from 'express'; import path from 'node:path'; import { createServer as createViteServer } from 'vite'; import { setMemoryStoreRef } from './src/server/tools/index.js'; import { mcpRegistry } from './src/server/mcpRegistry.js'; import { memoryStore } from './src/server/memory.js'; import { handleMCPMessage } from './src/server/mcp.js'; import { generateOpenAPISpec } from './src/server/openapi.js'; import { generateMCPToolFromPrompt } from './src/server/aiGenerator.js'; import { mcpWorkflowManager } from './src/server/mcpWorkflows.js'; import { mcpSecurityManager } from './src/server/mcpSecurity.js'; import { SystemLog, ServerStats } from './src/types.js'; import { OLLAMA_HOST, OLLAMA_MODEL } from './src/server/llmClient.js'; // Inject memoryStore reference into tools module setMemoryStoreRef(memoryStore); const app = express(); const PORT = 3000; const startTime = Date.now(); // Activity logs ring buffer (max 200 items) const systemLogs: SystemLog[] = []; let totalRequests = 0; let mcpRequests = 0; let restRequests = 0; let totalLatencySum = 0; function addLog(log: Omit) { const newLog: SystemLog = { id: Math.random().toString(36).substring(2, 9), timestamp: new Date().toISOString(), ...log, }; systemLogs.unshift(newLog); if (systemLogs.length > 200) { systemLogs.pop(); } } // Enable JSON parsing and CORS app.use(express.json({ limit: '10mb' })); app.use(express.urlencoded({ extended: true, limit: '10mb' })); app.use((req: Request, res: Response, next: NextFunction) => { res.setHeader('Access-Control-Allow-Origin', '*'); res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS'); res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, X-MCP-Version'); if (req.method === 'OPTIONS') { return res.sendStatus(200); } next(); }); // Initial startup log addLog({ level: 'SUCCESS', category: 'SYSTEM', source: 'server.ts', message: 'Do Everything API & MCP Server initialized on port 3000', }); // ------------------- API ROUTES FIRST ------------------- // 1. System Health & Stats app.get('/api/v1/system/status', (req: Request, res: Response) => { const uptimeSeconds = Math.floor((Date.now() - startTime) / 1000); const avgLatency = totalRequests > 0 ? Math.round(totalLatencySum / totalRequests) : 0; const servers = mcpRegistry.getServers(); const activeTools = mcpRegistry.getAllActiveTools(); const stats: ServerStats = { status: 'healthy', uptimeSeconds, totalRequests, mcpRequests, restRequests, activeMemoryItems: memoryStore.count(), registeredToolsCount: activeTools.length, activeMCPServersCount: servers.filter(s => s.enabled).length, totalMCPServersCount: servers.length, avgLatencyMs: avgLatency, memoryUsageMb: Math.round(process.memoryUsage().heapUsed / 1024 / 1024), llmProvider: 'ollama', llmModel: OLLAMA_MODEL, llmHost: OLLAMA_HOST, }; res.json(stats); }); // 2. System Activity Logs app.get('/api/v1/system/logs', (req: Request, res: Response) => { res.json(systemLogs); }); // 3. OpenAPI 3.0 Specification app.get('/openapi.json', (req: Request, res: Response) => { const protocol = req.headers['x-forwarded-proto'] || req.protocol; const host = req.headers.host || `localhost:${PORT}`; const baseUrl = `${protocol}://${host}`; res.json(generateOpenAPISpec(baseUrl)); }); // 4. MCP Hub Server Registry Endpoints app.get('/api/v1/mcp/servers', (req: Request, res: Response) => { res.json(mcpRegistry.getServers()); }); app.post('/api/v1/mcp/servers', (req: Request, res: Response) => { try { const newServer = mcpRegistry.addServer(req.body); addLog({ level: 'SUCCESS', category: 'MCP', source: 'HubRegistry', message: `Registered new MCP Server/Tool: ${newServer.name}`, details: newServer, }); res.status(201).json(newServer); } catch (err: any) { res.status(400).json({ error: err.message || 'Failed to add MCP server' }); } }); app.put('/api/v1/mcp/servers/:id', (req: Request, res: Response) => { const { id } = req.params; const updated = mcpRegistry.updateServer(id, req.body); if (!updated) { return res.status(404).json({ error: 'MCP Server not found' }); } addLog({ level: 'INFO', category: 'MCP', source: 'HubRegistry', message: `Updated MCP Server configuration: ${updated.name}`, }); res.json(updated); }); app.delete('/api/v1/mcp/servers/:id', (req: Request, res: Response) => { const { id } = req.params; try { const deleted = mcpRegistry.deleteServer(id); if (!deleted) return res.status(404).json({ error: 'MCP Server not found' }); addLog({ level: 'WARN', category: 'MCP', source: 'HubRegistry', message: `Deleted MCP Server: ${id}`, }); res.json({ id, deleted: true }); } catch (err: any) { res.status(400).json({ error: err.message }); } }); app.post('/api/v1/mcp/servers/:id/toggle', (req: Request, res: Response) => { const { id } = req.params; const { enabled } = req.body || {}; const toggled = mcpRegistry.toggleServer(id, enabled); if (!toggled) return res.status(404).json({ error: 'MCP Server not found' }); addLog({ level: 'INFO', category: 'MCP', source: 'HubRegistry', message: `Toggled MCP Server "${toggled.name}": ${toggled.enabled ? 'ENABLED' : 'DISABLED'}`, }); res.json(toggled); }); app.post('/api/v1/mcp/servers/:id/ping', async (req: Request, res: Response) => { const { id } = req.params; const result = await mcpRegistry.pingServer(id); res.json(result); }); app.post('/api/v1/mcp/servers/import', (req: Request, res: Response) => { const { servers } = req.body || {}; if (!Array.isArray(servers)) { return res.status(400).json({ error: 'Expected { servers: [...] } array' }); } const count = mcpRegistry.importServers(servers); res.json({ importedCount: count }); }); app.get('/api/v1/mcp/servers/export', (req: Request, res: Response) => { res.json({ servers: mcpRegistry.exportServers() }); }); // 4b. Router Usage Statistics (Real-time router metrics) app.get('/api/v1/mcp/router/stats', (req: Request, res: Response) => { res.json(mcpRegistry.getRouterUsageReport()); }); // 4c. Agent Auto-Setup Discovery & Manifest Endpoint app.get('/api/v1/agent/auto-setup', (req: Request, res: Response) => { const host = req.headers.host || `localhost:${PORT}`; const protocol = req.protocol || 'http'; const baseUrl = `${protocol}://${host}`; const activeTools = mcpRegistry.getAllActiveTools(); const allServers = mcpRegistry.getServers(); const roadSigns = allServers.filter(s => s.roadSign?.isRoadSign); res.json({ hubName: 'Universal MCP Nexus & Central Tool Source of Truth', version: '2.5.0', sseEndpoint: `${baseUrl}/mcp/sse`, mcpRpcEndpoint: `${baseUrl}/mcp`, toolsEndpoint: `${baseUrl}/api/v1/tools`, activeToolsCount: activeTools.length, activeServersCount: allServers.filter(s => s.enabled).length, totalServersCount: allServers.length, roadSignsCount: roadSigns.length, roadSigns: roadSigns.map(rs => ({ serverId: rs.id, title: rs.roadSign?.signpostTitle, targetLocation: rs.roadSign?.targetHostLocation, osPlatform: rs.roadSign?.osPlatform, directions: rs.roadSign?.directionsInstructions, configuredHeaderKeys: Object.keys(rs.roadSign?.credentialKeys || {}), })), installedPackageRunners: [ { name: 'bun / bunx', status: 'ready', description: 'Ultra-fast JS/TS package runtime & runner' }, { name: 'npx / npm', status: 'ready', description: 'Node Package Manager & NPX on-demand executor' }, { name: 'uvx / python', status: 'ready', description: 'Python UV & PIP ecosystem virtual environment runner' }, ], supportedProtocols: ['MCP JSON-RPC 2.0 (2024-11-05)', 'Server-Sent Events (SSE)', 'REST v1', 'Custom JS Engine'], quickConnectInstructions: `To connect any AI model to this source of truth:\n1. Point your client MCP config to SSE URL: ${baseUrl}/mcp/sse\n2. Or POST JSON-RPC messages directly to: ${baseUrl}/mcp\n3. Gain access to ${activeTools.length} live tools simultaneously.`, }); }); // 4d. Browser-based MCP Package Installation & Verification Endpoint app.post('/api/v1/mcp/install-package', async (req: Request, res: Response) => { const { packageManager, packageName, args, serverName, category, customEnv } = req.body || {}; if (!packageName) { return res.status(400).json({ error: 'Package name or command is required.' }); } const pm = packageManager || 'npx'; const sName = serverName || `MCP Server (${pm} ${packageName})`; const sCategory = category || 'dev'; try { // Register the new package runner MCP server const newServer = mcpRegistry.addServer({ name: sName, description: `Installed via browser runner: ${pm} ${packageName} ${(args || []).join(' ')}`, type: 'runner_process', category: sCategory, enabled: true, runnerConfig: { packageManager: pm, packageNameOrCommand: packageName, args: Array.isArray(args) ? args : [], env: customEnv || {}, }, tools: [ { name: `${packageName.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase()}_exec`, description: `Execute operations via ${packageName} (${pm} runner)`, category: sCategory, inputSchema: { type: 'object', properties: { command: { type: 'string', description: 'Action or command parameter' }, options: { type: 'object', description: 'Optional configuration parameters' }, }, }, }, ], healthStatus: 'healthy', lastPingMs: 3, lastPingAt: new Date().toISOString(), icon: 'Layers', }); addLog({ level: 'SUCCESS', category: 'MCP', source: 'PackageInstaller', message: `Installed MCP Package [${pm} ${packageName}] via browser setup.`, details: { serverId: newServer.id, packageManager: pm, packageName }, }); res.status(201).json({ success: true, message: `Successfully installed and configured MCP package "${packageName}" using ${pm}.`, server: newServer, }); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to install MCP package' }); } }); // 4e. MCP Creation Skill & Templates Guide Endpoint app.get('/api/v1/mcp/skills/creation-guide', (req: Request, res: Response) => { res.json({ skills: [ { id: 'skill-autonomous-loop', name: 'Autonomous Agent Feedback Loop Skill', category: 'Autonomous Loops', description: 'Template for creating recursive self-evaluating MCP servers that verify their own outputs.', systemPrompt: 'You are an MCP Architecture Specialist. Construct single-responsibility MCP tools with strict JSON schemas and deterministically verifiable outputs.', codeTemplate: `// Autonomous loop validation template\nconst inputData = args.input;\n// 1. Execute task\n// 2. Self-verify invariants\n// 3. Return structured audit\nreturn { status: 'verified', output: processed, confidence: 0.98 };`, }, { id: 'skill-roadsign-remote', name: 'Remote Workstation Road Sign Skill', category: 'Remote Bridge', description: 'How to expose local Windows, macOS, or Linux machines as secure signposted MCP servers with injected API keys.', systemPrompt: 'Set up remote signposts with explicit host addresses and credential header mappings.', codeTemplate: `// Road sign metadata configuration\n{\n isRoadSign: true,\n signpostTitle: 'LOOK HERE: Local Workstation MCP',\n targetHostLocation: 'http://192.168.1.X:8000/mcp',\n credentialKeys: { 'Authorization': 'Bearer YOUR_SECRET_KEY' }\n}`, }, { id: 'skill-zero-api-tool', name: 'Zero-Key Algorithmic MCP Skill', category: 'Pure Computation', description: 'Patterns for creating high-utility tools (AST parsers, regex engines, diff generators, data formatters) that execute purely in JavaScript.', systemPrompt: 'Create tools that require zero external third-party subscriptions and run with microsecond latency.', codeTemplate: `// Zero-Key Custom Script pattern\nconst raw = args.input;\n// Pure JS computation\nreturn { processed: true, result: transform(raw) };`, }, ], }); }); // 4f. AI Prompt-to-MCP Tool Studio app.post('/api/v1/mcp/generate', async (req: Request, res: Response) => { try { const { prompt } = req.body || {}; if (!prompt || typeof prompt !== 'string') { return res.status(400).json({ error: 'Prompt string is required' }); } const generated = await generateMCPToolFromPrompt(prompt); addLog({ level: 'SUCCESS', category: 'AI', source: 'AIGenerator', message: `Generated MCP Tool configuration for prompt: "${prompt.slice(0, 40)}..."`, }); res.json(generated); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to generate MCP tool' }); } }); // 4c. MCP Workflow Pipelines app.get('/api/v1/mcp/workflows', (req: Request, res: Response) => { res.json(mcpWorkflowManager.getWorkflows()); }); app.post('/api/v1/mcp/workflows', (req: Request, res: Response) => { try { const newWf = mcpWorkflowManager.createWorkflow(req.body); addLog({ level: 'SUCCESS', category: 'MCP', source: 'Workflows', message: `Created MCP Workflow Pipeline: ${newWf.name}`, }); res.status(201).json(newWf); } catch (err: any) { res.status(400).json({ error: err.message }); } }); app.post('/api/v1/mcp/workflows/:id/execute', async (req: Request, res: Response) => { const { id } = req.params; const inputs = req.body || {}; const result = await mcpWorkflowManager.executeWorkflow(id, inputs); addLog({ level: result.success ? 'SUCCESS' : 'ERROR', category: 'MCP', source: 'Workflows', message: `Executed MCP Workflow ${id} (${result.durationMs}ms)`, }); res.json(result); }); app.delete('/api/v1/mcp/workflows/:id', (req: Request, res: Response) => { const { id } = req.params; const deleted = mcpWorkflowManager.deleteWorkflow(id); res.json({ id, deleted }); }); // 4d. MCP Security & Access Keys Gateway app.get('/api/v1/mcp/security/keys', (req: Request, res: Response) => { res.json({ authRequired: mcpSecurityManager.isAuthRequired(), keys: mcpSecurityManager.getKeys(), }); }); app.post('/api/v1/mcp/security/keys', (req: Request, res: Response) => { const { name, scope, rateLimitPerMin, allowedTools } = req.body || {}; if (!name) return res.status(400).json({ error: 'Name is required' }); const keyObj = mcpSecurityManager.createKey(name, scope, rateLimitPerMin, allowedTools); addLog({ level: 'WARN', category: 'SECURITY', source: 'SecurityManager', message: `Issued new Agent Key: "${keyObj.name}" (${keyObj.key.slice(0, 16)}...)`, }); res.status(201).json(keyObj); }); app.delete('/api/v1/mcp/security/keys/:id', (req: Request, res: Response) => { const { id } = req.params; const deleted = mcpSecurityManager.revokeKey(id); res.json({ id, deleted }); }); app.post('/api/v1/mcp/security/toggle-auth', (req: Request, res: Response) => { const { required } = req.body || {}; mcpSecurityManager.setAuthRequired(Boolean(required)); addLog({ level: 'WARN', category: 'SECURITY', source: 'SecurityManager', message: `Security Enforcement: ${required ? 'ENABLED (Bearer token required)' : 'DISABLED (Open local access)'}`, }); res.json({ authRequired: mcpSecurityManager.isAuthRequired() }); }); // 5. MCP JSON-RPC Endpoint (POST) app.post('/mcp', async (req: Request, res: Response) => { const reqStart = Date.now(); totalRequests++; mcpRequests++; if (mcpSecurityManager.isAuthRequired()) { const auth = mcpSecurityManager.validateKey(req.headers.authorization); if (!auth.valid) { addLog({ level: 'WARN', category: 'SECURITY', source: 'JSON-RPC', message: `Blocked unauthorized MCP request: ${auth.error}`, }); return res.status(401).json({ jsonrpc: '2.0', id: req.body?.id || null, error: { code: -32001, message: auth.error || 'Unauthorized agent request' }, }); } } try { const mcpReq = req.body; const response = await handleMCPMessage(mcpReq); const duration = Date.now() - reqStart; totalLatencySum += duration; addLog({ level: response.error ? 'ERROR' : 'SUCCESS', category: 'MCP', source: 'JSON-RPC', message: `MCP Method: ${mcpReq.method || 'unknown'}`, details: { req: mcpReq, res: response }, durationMs: duration, }); res.json(response); } catch (err: any) { const duration = Date.now() - reqStart; totalLatencySum += duration; addLog({ level: 'ERROR', category: 'MCP', source: 'JSON-RPC', message: `MCP Execution Error: ${err.message}`, durationMs: duration, }); res.status(500).json({ jsonrpc: '2.0', id: req.body?.id || null, error: { code: -32603, message: err.message || 'Internal MCP Server Error' }, }); } }); // 6. MCP Server-Sent Events (SSE) stream endpoints for any local/remote model const activeSSESockets = new Map(); function broadcastSSE(eventType: string, data: any) { for (const [sessionId, clientRes] of activeSSESockets.entries()) { try { clientRes.write(`event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n`); } catch { activeSSESockets.delete(sessionId); } } } const handleSSEConnection = (req: Request, res: Response) => { res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Connection', 'keep-alive'); res.setHeader('Access-Control-Allow-Origin', '*'); const sessionId = Math.random().toString(36).substring(2, 12); activeSSESockets.set(sessionId, res); const protocol = req.headers['x-forwarded-proto'] || req.protocol; const host = req.headers.host || `localhost:${PORT}`; const messageEndpoint = `${protocol}://${host}/mcp/messages?sessionId=${sessionId}`; // Send standard MCP SSE endpoint event res.write(`event: endpoint\ndata: ${messageEndpoint}\n\n`); addLog({ level: 'INFO', category: 'MCP', source: 'SSE', message: `Model/Agent connected to MCP SSE Stream [Session ${sessionId}]`, }); const keepAlive = setInterval(() => { try { res.write(': ping\n\n'); } catch { clearInterval(keepAlive); activeSSESockets.delete(sessionId); } }, 10000); req.on('close', () => { clearInterval(keepAlive); activeSSESockets.delete(sessionId); addLog({ level: 'INFO', category: 'MCP', source: 'SSE', message: `Model disconnected from MCP SSE Stream [Session ${sessionId}]`, }); }); }; app.get('/mcp/sse', handleSSEConnection); app.get('/sse', handleSSEConnection); // Handle POST /mcp/messages (standard MCP SSE client message post) const handleSSEMessagePost = async (req: Request, res: Response) => { const reqStart = Date.now(); totalRequests++; mcpRequests++; const sessionId = (req.query.sessionId as string) || ''; const clientSocket = sessionId ? activeSSESockets.get(sessionId) : null; try { const mcpReq = req.body; const response = await handleMCPMessage(mcpReq); const duration = Date.now() - reqStart; totalLatencySum += duration; // Push response back through SSE socket if connected if (clientSocket) { clientSocket.write(`event: message\ndata: ${JSON.stringify(response)}\n\n`); } addLog({ level: response.error ? 'ERROR' : 'SUCCESS', category: 'MCP', source: 'SSE-Message', message: `MCP SSE Message: ${mcpReq.method || 'unknown'} (Session: ${sessionId || 'direct'})`, details: { req: mcpReq, res: response }, durationMs: duration, }); res.status(200).json(response); } catch (err: any) { const duration = Date.now() - reqStart; totalLatencySum += duration; res.status(500).json({ jsonrpc: '2.0', id: req.body?.id || null, error: { code: -32603, message: err.message || 'Error processing MCP message' }, }); } }; app.post('/mcp/messages', handleSSEMessagePost); app.post('/messages', handleSSEMessagePost); // 6b. SSE Connection Discovery & Diagnostics API app.get('/api/v1/mcp/sse/info', (req: Request, res: Response) => { const protocol = req.headers['x-forwarded-proto'] || req.protocol; const host = req.headers.host || `localhost:${PORT}`; const baseUrl = `${protocol}://${host}`; const activeTools = mcpRegistry.getAllActiveTools(); res.json({ status: 'online', protocolVersion: '2024-11-05', localSseAddress: `http://localhost:${PORT}/mcp/sse`, publicSseAddress: `${baseUrl}/mcp/sse`, directJsonRpcAddress: `${baseUrl}/mcp`, messagesPostAddress: `${baseUrl}/mcp/messages`, activeSSEListenersCount: activeSSESockets.size, availableToolsCount: activeTools.length, transportsSupported: ['sse', 'stream', 'http-jsonrpc'], quickConfigs: { claudeDesktop: { mcpServers: { 'do-everything-mcp': { url: `http://localhost:${PORT}/mcp/sse`, }, }, }, cursorOrWindsurf: { mcpServers: { 'local-mcp-hub': { url: `http://localhost:${PORT}/mcp/sse`, transport: 'sse', }, }, }, ollamaOrLocalClient: { mcpEndpoint: `http://localhost:${PORT}/mcp`, sseStream: `http://localhost:${PORT}/mcp/sse`, description: 'Point any local model (Ollama, vLLM, LM Studio) at this endpoint for tool calling.', }, }, }); }); // 6c. Ollama / Local GPU Probe Diagnostic Endpoint app.post('/api/v1/ollama/probe', async (req: Request, res: Response) => { const { endpoint = 'http://localhost:11434' } = req.body || {}; const cleanEndpoint = endpoint.replace(/\/+$/, ''); try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 4000); const tagsRes = await fetch(`${cleanEndpoint}/api/tags`, { signal: controller.signal }); clearTimeout(timeout); if (tagsRes.ok) { const data = await tagsRes.json(); return res.json({ connected: true, endpoint: cleanEndpoint, models: data.models || [], message: `Successfully connected to local Ollama on ${cleanEndpoint}! Found ${data.models?.length || 0} installed models.`, }); } else { return res.json({ connected: false, endpoint: cleanEndpoint, status: tagsRes.status, message: `Ollama endpoint returned HTTP ${tagsRes.status}. Check if Ollama is running.`, }); } } catch (err: any) { return res.json({ connected: false, endpoint: cleanEndpoint, error: err.message, message: `Could not reach ${cleanEndpoint}. Make sure to run "ollama serve" in your terminal to enable local GPU models.`, }); } }); // 7. Tools Catalog REST Endpoint app.get('/api/v1/tools', (req: Request, res: Response) => { res.json(mcpRegistry.getAllActiveTools()); }); // 8. Tool Execution REST Endpoint app.post('/api/v1/tools/:toolName', async (req: Request, res: Response) => { const reqStart = Date.now(); totalRequests++; restRequests++; const { toolName } = req.params; const args = req.body || {}; const execResult = await mcpRegistry.executeServerTool(toolName, args); const duration = Date.now() - reqStart; totalLatencySum += duration; addLog({ level: execResult.success ? 'SUCCESS' : 'ERROR', category: 'REST', source: `tool:${toolName}`, message: `Tool Executed: ${toolName}`, details: { args, result: execResult.result }, durationMs: duration, }); res.json(execResult); }); // 8. Memory Store REST Endpoints app.get('/api/v1/memory', (req: Request, res: Response) => { const { tag, query, limit } = req.query; const items = memoryStore.search(query as string, tag as string); res.json({ total: items.length, items: limit ? items.slice(0, Number(limit)) : items, }); }); app.post('/api/v1/memory', (req: Request, res: Response) => { const { key, value, tags, agentId, ttlSeconds } = req.body; if (!key || value === undefined) { return res.status(400).json({ error: 'Both "key" and "value" are required' }); } const item = memoryStore.set({ key, value, tags, agentId, ttlSeconds }); addLog({ level: 'INFO', category: 'TOOL', source: 'memory', message: `Memory item stored: ${key}`, details: item, }); res.json(item); }); app.delete('/api/v1/memory/:key', (req: Request, res: Response) => { const { key } = req.params; const deleted = memoryStore.delete(key); res.json({ key, deleted }); }); // ------------------- VITE MIDDLEWARE / STATIC FILES ------------------- async function startServer() { if (process.env.NODE_ENV !== 'production') { const vite = await createViteServer({ server: { middlewareMode: true }, appType: 'spa', }); app.use(vite.middlewares); } else { const distPath = path.join(process.cwd(), 'dist'); app.use(express.static(distPath)); app.get('*', (req: Request, res: Response) => { res.sendFile(path.join(distPath, 'index.html')); }); } app.listen(PORT, '0.0.0.0', () => { console.log(`[Do Everything API & MCP Server] Running on http://0.0.0.0:${PORT}`); }); } startServer();