From d04fe5a3ca772392373cfef2bbc48db61477bee3 Mon Sep 17 00:00:00 2001 From: drjones Date: Tue, 25 Aug 2026 06:43:46 +0000 Subject: [PATCH] assimilate 1mcpserver: MCP Discovery module (search/readme/stats over 1007-server catalog) --- src/server/tools/index.ts | 8 ++ src/server/tools/mcpDiscovery.ts | 226 +++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 src/server/tools/mcpDiscovery.ts diff --git a/src/server/tools/index.ts b/src/server/tools/index.ts index 9ab5ca6..0795c9f 100644 --- a/src/server/tools/index.ts +++ b/src/server/tools/index.ts @@ -2,6 +2,7 @@ import crypto from 'node:crypto'; import { callLocalModel } from '../llmClient.js'; import { ToolDefinition, ToolExecutionResult } from '../../types.js'; import { LOCAL_MODEL_TOOLS, executeLocalModelTool } from './localModelTools.js'; +import { MCP_DISCOVERY_TOOLS, executeMcpDiscoveryTool } from './mcpDiscovery.js'; // Memory Store reference will be imported lazily or injected let memoryStoreRef: any = null; @@ -13,6 +14,9 @@ export const TOOLS: ToolDefinition[] = [ // Local Ollama & NVIDIA GPU Delegation Tools ...LOCAL_MODEL_TOOLS, + // MCP Discovery (assimilated from particlefuture/1mcpserver) + ...MCP_DISCOVERY_TOOLS, + // 1. WEB SCRAPER MARKDOWN { name: 'web_scrape_markdown', @@ -1024,6 +1028,10 @@ export async function executeTool(toolName: string, args: any): Promise t.name === toolName); + if (isDiscoveryTool) { + return await executeMcpDiscoveryTool(toolName, args); + } throw new Error(`Tool "${toolName}" not found or not supported.`); } } diff --git a/src/server/tools/mcpDiscovery.ts b/src/server/tools/mcpDiscovery.ts new file mode 100644 index 0000000..63e02ab --- /dev/null +++ b/src/server/tools/mcpDiscovery.ts @@ -0,0 +1,226 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { ToolDefinition, ToolExecutionResult } from '../../types.js'; + +// --------------------------------------------------------------------------- +// MCP Discovery module — assimilated from particlefuture/1mcpserver +// Provides semantic-ish keyword search + README lookup over a curated +// catalog of 1000+ MCP servers (name, description, GitHub URL, API-key flag). +// --------------------------------------------------------------------------- + +interface McpCatalogEntry { + name: string; + description: string; + url: string; + requires_api_key: boolean; +} + +const DATA_DIR = process.env.MCP_DATA_DIR || path.resolve(process.cwd(), 'data'); +const CATALOG_PATH = path.join(DATA_DIR, 'mcp_catalog.json'); +const READMES_PATH = path.join(DATA_DIR, 'mcp_readmes.json'); + +let catalog: McpCatalogEntry[] = []; +let readmes: Record | null = null; +let loadError: string | null = null; + +function loadCatalog(): void { + if (catalog.length > 0 || loadError) return; + try { + if (!fs.existsSync(CATALOG_PATH)) { + loadError = `Catalog file not found at ${CATALOG_PATH}`; + return; + } + catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf-8')); + } catch (e: any) { + loadError = `Failed to load MCP catalog: ${e.message}`; + } +} + +function loadReadmes(): void { + if (readmes) return; + try { + readmes = fs.existsSync(READMES_PATH) + ? JSON.parse(fs.readFileSync(READMES_PATH, 'utf-8')) + : {}; + } catch { + readmes = {}; + } +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function searchCatalog(query: string, limit: number): any[] { + loadCatalog(); + const q = (query || '').toLowerCase().trim(); + if (!q) return []; + const tokens = q.split(/\s+/).filter(Boolean); + + const scored = catalog + .map((entry) => { + const name = (entry.name || '').toLowerCase(); + const desc = (entry.description || '').toLowerCase(); + let score = 0; + for (const t of tokens) { + if (name === t) score += 100; + else if (name.includes(t)) score += 40; + if (desc.includes(t)) score += 6; + try { + if (new RegExp(`\\b${escapeRegExp(t)}`).test(name)) score += 10; + } catch { + /* skip malformed regex */ + } + } + if (tokens.every((t) => name.includes(t) || desc.includes(t))) score += 5; + return { entry, score }; + }) + .filter((x) => x.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, limit || 10); + + return scored.map((x) => ({ + name: x.entry.name, + description: x.entry.description, + url: x.entry.url, + requires_api_key: x.entry.requires_api_key, + score: x.score, + })); +} + +export const MCP_DISCOVERY_TOOLS: ToolDefinition[] = [ + { + name: 'mcp_discovery_search', + description: + 'Search a curated catalog of 1000+ MCP servers by keyword. Returns ranked matches (name, description, GitHub URL, whether an API key is required). Use to discover MCP servers/tools for any capability.', + category: 'mcp', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: + 'Free-text query describing the capability you need (e.g. "payments", "browser automation", "sql database", "image generation").', + }, + limit: { + type: 'number', + description: 'Max results to return (default 10, max 50).', + }, + }, + required: ['query'], + }, + }, + { + name: 'mcp_discovery_readme', + description: + 'Fetch the README (setup/config instructions) for an MCP server in the catalog by its name or GitHub URL.', + category: 'mcp', + inputSchema: { + type: 'object', + properties: { + name: { + type: 'string', + description: 'Server name or GitHub URL to look up.', + }, + }, + required: ['name'], + }, + }, + { + name: 'mcp_discovery_stats', + description: + 'Return stats about the MCP server catalog (total servers, how many require API keys, source).', + category: 'mcp', + inputSchema: { + type: 'object', + properties: {}, + }, + }, +]; + +export async function executeMcpDiscoveryTool( + toolName: string, + args: any, +): Promise { + const startTime = Date.now(); + try { + let result: any; + let contentType: ToolExecutionResult['contentType'] = 'json'; + + switch (toolName) { + case 'mcp_discovery_search': { + const { query, limit = 10 } = args; + const matches = searchCatalog(query, Math.min(limit || 10, 50)); + if (!matches.length) { + result = { + query, + total: 0, + results: [], + hint: 'No matches. Try broader or different keywords.', + }; + } else { + result = { query, total: matches.length, results: matches }; + } + break; + } + + case 'mcp_discovery_readme': { + loadCatalog(); + loadReadmes(); + const needle = String(args.name || '').trim().toLowerCase(); + if (!needle) throw new Error('name is required'); + const entry = catalog.find( + (e) => + (e.name || '').toLowerCase() === needle || + (e.url || '').toLowerCase() === needle || + (e.url || '').toLowerCase().includes(needle), + ); + if (!entry) throw new Error(`No catalog entry matching "${args.name}"`); + const readme = readmes?.[entry.url] || ''; + result = { + name: entry.name, + url: entry.url, + description: entry.description, + requires_api_key: entry.requires_api_key, + readme: readme + ? readme.slice(0, 12000) + : '(no README cached — fetch from the GitHub URL directly)', + }; + contentType = 'markdown'; + break; + } + + case 'mcp_discovery_stats': { + loadCatalog(); + const total = catalog.length; + const needKey = catalog.filter((e) => e.requires_api_key).length; + result = { + total_servers: total, + requires_api_key: needKey, + no_key_required: total - needKey, + source: '1mcpserver catalog (particlefuture/1mcpserver)', + }; + break; + } + + default: + throw new Error(`Tool "${toolName}" not supported by MCP Discovery module`); + } + + return { + success: true, + toolName, + result, + executionTimeMs: Date.now() - startTime, + contentType, + }; + } catch (err: any) { + return { + success: false, + toolName, + result: null, + error: err.message || 'MCP discovery failed', + executionTimeMs: Date.now() - startTime, + }; + } +}