assimilate 1mcpserver: MCP Discovery module (search/readme/stats over 1007-server catalog)

src/server/tools/mcpDiscovery.ts

@@ -0,0 +1,226 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ToolDefinition, ToolExecutionResult } from '../../types.js';
+
+// ---------------------------------------------------------------------------
+// MCP Discovery module — assimilated from particlefuture/1mcpserver
+// Provides semantic-ish keyword search + README lookup over a curated
+// catalog of 1000+ MCP servers (name, description, GitHub URL, API-key flag).
+// ---------------------------------------------------------------------------
+
+interface McpCatalogEntry {
+  name: string;
+  description: string;
+  url: string;
+  requires_api_key: boolean;
+}
+
+const DATA_DIR = process.env.MCP_DATA_DIR || path.resolve(process.cwd(), 'data');
+const CATALOG_PATH = path.join(DATA_DIR, 'mcp_catalog.json');
+const READMES_PATH = path.join(DATA_DIR, 'mcp_readmes.json');
+
+let catalog: McpCatalogEntry[] = [];
+let readmes: Record<string, string> | null = null;
+let loadError: string | null = null;
+
+function loadCatalog(): void {
+  if (catalog.length > 0 || loadError) return;
+  try {
+    if (!fs.existsSync(CATALOG_PATH)) {
+      loadError = `Catalog file not found at ${CATALOG_PATH}`;
+      return;
+    }
+    catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf-8'));
+  } catch (e: any) {
+    loadError = `Failed to load MCP catalog: ${e.message}`;
+  }
+}
+
+function loadReadmes(): void {
+  if (readmes) return;
+  try {
+    readmes = fs.existsSync(READMES_PATH)
+      ? JSON.parse(fs.readFileSync(READMES_PATH, 'utf-8'))
+      : {};
+  } catch {
+    readmes = {};
+  }
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function searchCatalog(query: string, limit: number): any[] {
+  loadCatalog();
+  const q = (query || '').toLowerCase().trim();
+  if (!q) return [];
+  const tokens = q.split(/\s+/).filter(Boolean);
+
+  const scored = catalog
+    .map((entry) => {
+      const name = (entry.name || '').toLowerCase();
+      const desc = (entry.description || '').toLowerCase();
+      let score = 0;
+      for (const t of tokens) {
+        if (name === t) score += 100;
+        else if (name.includes(t)) score += 40;
+        if (desc.includes(t)) score += 6;
+        try {
+          if (new RegExp(`\\b${escapeRegExp(t)}`).test(name)) score += 10;
+        } catch {
+          /* skip malformed regex */
+        }
+      }
+      if (tokens.every((t) => name.includes(t) || desc.includes(t))) score += 5;
+      return { entry, score };
+    })
+    .filter((x) => x.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit || 10);
+
+  return scored.map((x) => ({
+    name: x.entry.name,
+    description: x.entry.description,
+    url: x.entry.url,
+    requires_api_key: x.entry.requires_api_key,
+    score: x.score,
+  }));
+}
+
+export const MCP_DISCOVERY_TOOLS: ToolDefinition[] = [
+  {
+    name: 'mcp_discovery_search',
+    description:
+      'Search a curated catalog of 1000+ MCP servers by keyword. Returns ranked matches (name, description, GitHub URL, whether an API key is required). Use to discover MCP servers/tools for any capability.',
+    category: 'mcp',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: {
+          type: 'string',
+          description:
+            'Free-text query describing the capability you need (e.g. "payments", "browser automation", "sql database", "image generation").',
+        },
+        limit: {
+          type: 'number',
+          description: 'Max results to return (default 10, max 50).',
+        },
+      },
+      required: ['query'],
+    },
+  },
+  {
+    name: 'mcp_discovery_readme',
+    description:
+      'Fetch the README (setup/config instructions) for an MCP server in the catalog by its name or GitHub URL.',
+    category: 'mcp',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        name: {
+          type: 'string',
+          description: 'Server name or GitHub URL to look up.',
+        },
+      },
+      required: ['name'],
+    },
+  },
+  {
+    name: 'mcp_discovery_stats',
+    description:
+      'Return stats about the MCP server catalog (total servers, how many require API keys, source).',
+    category: 'mcp',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+];
+
+export async function executeMcpDiscoveryTool(
+  toolName: string,
+  args: any,
+): Promise<ToolExecutionResult> {
+  const startTime = Date.now();
+  try {
+    let result: any;
+    let contentType: ToolExecutionResult['contentType'] = 'json';
+
+    switch (toolName) {
+      case 'mcp_discovery_search': {
+        const { query, limit = 10 } = args;
+        const matches = searchCatalog(query, Math.min(limit || 10, 50));
+        if (!matches.length) {
+          result = {
+            query,
+            total: 0,
+            results: [],
+            hint: 'No matches. Try broader or different keywords.',
+          };
+        } else {
+          result = { query, total: matches.length, results: matches };
+        }
+        break;
+      }
+
+      case 'mcp_discovery_readme': {
+        loadCatalog();
+        loadReadmes();
+        const needle = String(args.name || '').trim().toLowerCase();
+        if (!needle) throw new Error('name is required');
+        const entry = catalog.find(
+          (e) =>
+            (e.name || '').toLowerCase() === needle ||
+            (e.url || '').toLowerCase() === needle ||
+            (e.url || '').toLowerCase().includes(needle),
+        );
+        if (!entry) throw new Error(`No catalog entry matching "${args.name}"`);
+        const readme = readmes?.[entry.url] || '';
+        result = {
+          name: entry.name,
+          url: entry.url,
+          description: entry.description,
+          requires_api_key: entry.requires_api_key,
+          readme: readme
+            ? readme.slice(0, 12000)
+            : '(no README cached — fetch from the GitHub URL directly)',
+        };
+        contentType = 'markdown';
+        break;
+      }
+
+      case 'mcp_discovery_stats': {
+        loadCatalog();
+        const total = catalog.length;
+        const needKey = catalog.filter((e) => e.requires_api_key).length;
+        result = {
+          total_servers: total,
+          requires_api_key: needKey,
+          no_key_required: total - needKey,
+          source: '1mcpserver catalog (particlefuture/1mcpserver)',
+        };
+        break;
+      }
+
+      default:
+        throw new Error(`Tool "${toolName}" not supported by MCP Discovery module`);
+    }
+
+    return {
+      success: true,
+      toolName,
+      result,
+      executionTimeMs: Date.now() - startTime,
+      contentType,
+    };
+  } catch (err: any) {
+    return {
+      success: false,
+      toolName,
+      result: null,
+      error: err.message || 'MCP discovery failed',
+      executionTimeMs: Date.now() - startTime,
+    };
+  }
+}