Fix Fleet AI and LOTL test regressions after parallel merges.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Stub slow syscheck/listen-port probes in agent tests, fix ai_snapshot mutex deadlock, reorder fleet clearance vs connectivity checks, and add AI control precedence plus LotlTimeline vitest coverage.
This commit is contained in:
@@ -3,6 +3,8 @@ package client
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"crypto-miner-agent/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidateAICommandPathRejectsTraversal(t *testing.T) {
|
func TestValidateAICommandPathRejectsTraversal(t *testing.T) {
|
||||||
@@ -118,3 +120,32 @@ func TestHandleAIRestartMining(t *testing.T) {
|
|||||||
t.Fatalf("action=%s ok=%v", gotAction, gotOK)
|
t.Fatalf("action=%s ok=%v", gotAction, gotOK)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandleAIFullSysCheck(t *testing.T) {
|
||||||
|
SetFullSysCheckCollector(func(_ config.RuntimeConfig, agentID string) *FullSysCheckReport {
|
||||||
|
return &FullSysCheckReport{
|
||||||
|
GeneratedAt: "2026-06-07T00:00:00Z",
|
||||||
|
AgentID: agentID,
|
||||||
|
Platform: "windows",
|
||||||
|
}
|
||||||
|
})
|
||||||
|
defer SetFullSysCheckCollector(nil)
|
||||||
|
|
||||||
|
var gotAction string
|
||||||
|
var gotOK bool
|
||||||
|
var gotBody string
|
||||||
|
c := newTestClient(t)
|
||||||
|
c.agentID = "agent-test-1"
|
||||||
|
c.commandResultHook = func(action string, success bool, message string) {
|
||||||
|
gotAction = action
|
||||||
|
gotOK = success
|
||||||
|
gotBody = message
|
||||||
|
}
|
||||||
|
c.handleAICommand("full_sys_check", 0, "", "", "")
|
||||||
|
if gotAction != "full_sys_check" || !gotOK {
|
||||||
|
t.Fatalf("action=%s ok=%v", gotAction, gotOK)
|
||||||
|
}
|
||||||
|
if !strings.Contains(gotBody, "generated_at") || !strings.Contains(gotBody, "agent-test-1") {
|
||||||
|
t.Fatalf("expected JSON report, got %q", gotBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ func (c *AgentClient) buildAISnapshot(miningHashrate float64) AISnapshot {
|
|||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
snap.ClearanceLevel = c.clearanceLevel
|
snap.ClearanceLevel = c.clearanceLevel
|
||||||
if len(c.atlasSkips) > 0 {
|
if len(c.atlasSkips) > 0 {
|
||||||
snap.AtlasSkips = c.atlasSkipsSnapshot()
|
snap.AtlasSkips = append([]AtlasSkip(nil), c.atlasSkips...)
|
||||||
}
|
}
|
||||||
if c.inheritedPhenotype != nil {
|
if c.inheritedPhenotype != nil {
|
||||||
copy := *c.inheritedPhenotype
|
copy := *c.inheritedPhenotype
|
||||||
|
|||||||
9
agent/client/listen_ports_hook.go
Normal file
9
agent/client/listen_ports_hook.go
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
// listenPortsCollector overrides collectListenPorts in tests. Nil restores platform defaults.
|
||||||
|
var listenPortsCollector func() *ListenPortsReport
|
||||||
|
|
||||||
|
// SetListenPortsCollector stubs listen-port collection in tests. Pass nil to restore defaults.
|
||||||
|
func SetListenPortsCollector(fn func() *ListenPortsReport) {
|
||||||
|
listenPortsCollector = fn
|
||||||
|
}
|
||||||
@@ -11,6 +11,9 @@ import (
|
|||||||
// collectListenPorts parses ss -tlnp output for all TCP listeners.
|
// collectListenPorts parses ss -tlnp output for all TCP listeners.
|
||||||
// Falls back to netstat -tlnp if ss is unavailable.
|
// Falls back to netstat -tlnp if ss is unavailable.
|
||||||
func collectListenPorts() *ListenPortsReport {
|
func collectListenPorts() *ListenPortsReport {
|
||||||
|
if listenPortsCollector != nil {
|
||||||
|
return listenPortsCollector()
|
||||||
|
}
|
||||||
r := &ListenPortsReport{}
|
r := &ListenPortsReport{}
|
||||||
|
|
||||||
// Prefer ss (iproute2) — faster and widely available on modern Linux
|
// Prefer ss (iproute2) — faster and widely available on modern Linux
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ import (
|
|||||||
// Uses Get-NetTCPConnection (fast, built into Win8+/2012+) with per-port
|
// Uses Get-NetTCPConnection (fast, built into Win8+/2012+) with per-port
|
||||||
// process name lookup via Get-Process.
|
// process name lookup via Get-Process.
|
||||||
func collectListenPorts() *ListenPortsReport {
|
func collectListenPorts() *ListenPortsReport {
|
||||||
|
if listenPortsCollector != nil {
|
||||||
|
return listenPortsCollector()
|
||||||
|
}
|
||||||
const script = `
|
const script = `
|
||||||
$ErrorActionPreference = 'SilentlyContinue'
|
$ErrorActionPreference = 'SilentlyContinue'
|
||||||
$procs = @{}
|
$procs = @{}
|
||||||
|
|||||||
@@ -63,18 +63,26 @@ func TestMiningDiagnosticsJSONShape(t *testing.T) {
|
|||||||
|
|
||||||
func TestInferMiningBlockersRemotePause(t *testing.T) {
|
func TestInferMiningBlockersRemotePause(t *testing.T) {
|
||||||
c := testDiagnosticsClient(t, config.RuntimeConfig{})
|
c := testDiagnosticsClient(t, config.RuntimeConfig{})
|
||||||
c.pool.PauseRemote()
|
d := MiningDiagnostics{
|
||||||
|
C2Connected: true,
|
||||||
d := c.collectMiningDiagnostics()
|
CPU: struct {
|
||||||
|
RemotePaused bool `json:"remote_paused"`
|
||||||
|
ScheduleBlocked bool `json:"schedule_blocked"`
|
||||||
|
ResourcesBlocked bool `json:"resources_blocked"`
|
||||||
|
HasJob bool `json:"has_job"`
|
||||||
|
Hashrate float64 `json:"hashrate_hps"`
|
||||||
|
}{RemotePaused: true},
|
||||||
|
}
|
||||||
|
blockers := c.inferMiningBlockers(d)
|
||||||
found := false
|
found := false
|
||||||
for _, b := range d.LikelyBlockers {
|
for _, b := range blockers {
|
||||||
if strings.Contains(b, "remote command") || strings.Contains(b, "container delegation") {
|
if strings.Contains(b, "remote command") || strings.Contains(b, "container delegation") {
|
||||||
found = true
|
found = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
t.Fatalf("expected remote pause blocker, got %v", d.LikelyBlockers)
|
t.Fatalf("expected remote pause blocker, got %v", blockers)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,9 @@ const syscheckRawMax = 12000
|
|||||||
|
|
||||||
// CollectFullSysCheck aggregates read-only host telemetry for the C2 dashboard.
|
// CollectFullSysCheck aggregates read-only host telemetry for the C2 dashboard.
|
||||||
func CollectFullSysCheck(cfg config.RuntimeConfig, agentID string) *FullSysCheckReport {
|
func CollectFullSysCheck(cfg config.RuntimeConfig, agentID string) *FullSysCheckReport {
|
||||||
|
if fullSysCheckCollector != nil {
|
||||||
|
return fullSysCheckCollector(cfg, agentID)
|
||||||
|
}
|
||||||
r := &FullSysCheckReport{
|
r := &FullSysCheckReport{
|
||||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||||
Platform: runtime.GOOS,
|
Platform: runtime.GOOS,
|
||||||
@@ -57,7 +60,11 @@ func CollectFullSysCheck(cfg config.RuntimeConfig, agentID string) *FullSysCheck
|
|||||||
arp := deploy.ArpNeighborIPs()
|
arp := deploy.ArpNeighborIPs()
|
||||||
r.Neighbors.ArpHosts = arp
|
r.Neighbors.ArpHosts = arp
|
||||||
r.Neighbors.ArpCount = len(arp)
|
r.Neighbors.ArpCount = len(arp)
|
||||||
r.Neighbors.SubnetScan = deploy.ScanLocalSubnet(20)
|
if subnetScanCollector != nil {
|
||||||
|
r.Neighbors.SubnetScan = subnetScanCollector(20)
|
||||||
|
} else {
|
||||||
|
r.Neighbors.SubnetScan = deploy.ScanLocalSubnet(20)
|
||||||
|
}
|
||||||
|
|
||||||
collectSysCheckPlatform(r)
|
collectSysCheckPlatform(r)
|
||||||
|
|
||||||
|
|||||||
19
agent/client/syscheck_hooks.go
Normal file
19
agent/client/syscheck_hooks.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
import "crypto-miner-agent/config"
|
||||||
|
|
||||||
|
// fullSysCheckCollector overrides CollectFullSysCheck in tests. Nil restores defaults.
|
||||||
|
var fullSysCheckCollector func(config.RuntimeConfig, string) *FullSysCheckReport
|
||||||
|
|
||||||
|
// subnetScanCollector overrides deploy.ScanLocalSubnet usage in syscheck. Nil restores defaults.
|
||||||
|
var subnetScanCollector func(maxHosts int) string
|
||||||
|
|
||||||
|
// SetFullSysCheckCollector stubs full_sys_check aggregation in tests.
|
||||||
|
func SetFullSysCheckCollector(fn func(config.RuntimeConfig, string) *FullSysCheckReport) {
|
||||||
|
fullSysCheckCollector = fn
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetSubnetScanCollector stubs subnet scan in syscheck tests.
|
||||||
|
func SetSubnetScanCollector(fn func(maxHosts int) string) {
|
||||||
|
subnetScanCollector = fn
|
||||||
|
}
|
||||||
@@ -9,6 +9,10 @@ import (
|
|||||||
func TestRunVulnLOTLProbeMockedContext(t *testing.T) {
|
func TestRunVulnLOTLProbeMockedContext(t *testing.T) {
|
||||||
SetPostureCollector(func() *PostureReport { return nil })
|
SetPostureCollector(func() *PostureReport { return nil })
|
||||||
defer SetPostureCollector(nil)
|
defer SetPostureCollector(nil)
|
||||||
|
SetListenPortsCollector(func() *ListenPortsReport {
|
||||||
|
return &ListenPortsReport{Ports: []ListenPort{{Port: 443, Proto: "tcp"}}}
|
||||||
|
})
|
||||||
|
defer SetListenPortsCollector(nil)
|
||||||
|
|
||||||
origProbe := vulnprobeProbeHost
|
origProbe := vulnprobeProbeHost
|
||||||
vulnprobeProbeHost = func(_ map[int]bool, _ string) vulnprobe.HostContext {
|
vulnprobeProbeHost = func(_ map[int]bool, _ string) vulnprobe.HostContext {
|
||||||
|
|||||||
@@ -27,6 +27,26 @@ func TestParseCommandsLineFallback(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseCommandsEmptyInput(t *testing.T) {
|
||||||
|
if cmds := ParseCommands(""); cmds != nil {
|
||||||
|
t.Fatalf("expected nil, got %+v", cmds)
|
||||||
|
}
|
||||||
|
if cmds := ParseCommands(" "); cmds != nil {
|
||||||
|
t.Fatalf("expected nil, got %+v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCommandsUnknownToolBecomesAgentCommand(t *testing.T) {
|
||||||
|
raw := `{"tool":"pause","args":{}}`
|
||||||
|
cmds := ParseCommands(raw)
|
||||||
|
if len(cmds) != 1 || cmds[0].Type != CmdAgentCommand {
|
||||||
|
t.Fatalf("got %+v", cmds)
|
||||||
|
}
|
||||||
|
if cmds[0].Args["action"] != "pause" {
|
||||||
|
t.Fatalf("args: %+v", cmds[0].Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseCommandsAgentCommand(t *testing.T) {
|
func TestParseCommandsAgentCommand(t *testing.T) {
|
||||||
raw := `{"commands":[{"type":"agent_command","args":{"action":"pause"}}]}`
|
raw := `{"commands":[{"type":"agent_command","args":{"action":"pause"}}]}`
|
||||||
cmds := ParseCommands(raw)
|
cmds := ParseCommands(raw)
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import (
|
|||||||
|
|
||||||
fleetai "crypto-miner-server/internal/ai"
|
fleetai "crypto-miner-server/internal/ai"
|
||||||
"crypto-miner-server/internal/db"
|
"crypto-miner-server/internal/db"
|
||||||
|
"crypto-miner-server/internal/models"
|
||||||
|
"crypto-miner-server/internal/strategy"
|
||||||
)
|
)
|
||||||
|
|
||||||
type stubFleetAIConfig struct {
|
type stubFleetAIConfig struct {
|
||||||
@@ -52,7 +54,7 @@ func TestFleetAIHandlerGetDecisions(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { database.Close() })
|
t.Cleanup(func() { database.Close() })
|
||||||
_ = database.InsertAIDecision("agent-x", "abc", `{"commands":[]}`, "noop:ok")
|
_ = database.InsertAIDecision("agent-x", "abc", `{"commands":[]}`, "noop:ok", false, "", "", "")
|
||||||
|
|
||||||
h := NewFleetAIHandler(nil, database)
|
h := NewFleetAIHandler(nil, database)
|
||||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/ai/decisions?agent_id=agent-x", nil)
|
req := httptest.NewRequest(http.MethodGet, "/api/v1/ai/decisions?agent_id=agent-x", nil)
|
||||||
@@ -70,6 +72,60 @@ func TestFleetAIHandlerGetDecisions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFleetAIHandlerGetModels(t *testing.T) {
|
||||||
|
modelSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"data": []map[string]string{{"id": "llama3.2"}},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
t.Cleanup(modelSrv.Close)
|
||||||
|
|
||||||
|
cfg := &stubFleetAIConfig{view: FleetAIConfigView{AIEndpoint: modelSrv.URL + "/v1"}}
|
||||||
|
h := NewFleetAIHandler(cfg, nil)
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/api/v1/ai/models", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.GetModels(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var body map[string]interface{}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
models, _ := body["models"].([]interface{})
|
||||||
|
if len(models) != 1 {
|
||||||
|
t.Fatalf("models: %v", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFleetAISnapshotOmitsAdaptiveWhenAIControl(t *testing.T) {
|
||||||
|
database, err := db.New(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
|
||||||
|
hub := NewWSHub(database)
|
||||||
|
hub.SetAdaptiveEngine(strategy.NewAdaptiveEngine(database, true))
|
||||||
|
hub.SetServerPolicy(ServerPolicy{AIControlEnabled: true})
|
||||||
|
agentID := "snap-agent-1"
|
||||||
|
_ = database.UpsertAgent(&models.Agent{
|
||||||
|
ID: agentID, Name: "node-a", Platform: "windows", Status: "online",
|
||||||
|
})
|
||||||
|
|
||||||
|
snap, ok := hub.FleetAISnapshot(agentID)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("snapshot not found")
|
||||||
|
}
|
||||||
|
if snap.Adaptive != nil {
|
||||||
|
t.Fatalf("adaptive must be nil when AI control enabled, got %+v", snap.Adaptive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFleetAIExecutorRestartMining(t *testing.T) {
|
func TestFleetAIExecutorRestartMining(t *testing.T) {
|
||||||
hub := NewWSHub(nil)
|
hub := NewWSHub(nil)
|
||||||
exec := &FleetAIExecutor{Hub: hub}
|
exec := &FleetAIExecutor{Hub: hub}
|
||||||
|
|||||||
@@ -386,6 +386,15 @@ func (f *FleetHandler) PostAgentCommand(w http.ResponseWriter, r *http.Request)
|
|||||||
http.Error(w, "websocket hub unavailable", http.StatusServiceUnavailable)
|
http.Error(w, "websocket hub unavailable", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if id != "all" && !f.ws.IsAgentReachable(id) {
|
||||||
|
writeJSON(w, map[string]interface{}{
|
||||||
|
"success": false,
|
||||||
|
"error": "agent not connected",
|
||||||
|
"agent_id": id,
|
||||||
|
"action": req.Action,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
if id != "all" {
|
if id != "all" {
|
||||||
level := clearance.L0
|
level := clearance.L0
|
||||||
if mgr := f.ws.ClearanceManager(); mgr != nil {
|
if mgr := f.ws.ClearanceManager(); mgr != nil {
|
||||||
@@ -428,15 +437,6 @@ func (f *FleetHandler) PostAgentCommand(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
f.ws.BroadcastAgentCommand(req.Action, args)
|
f.ws.BroadcastAgentCommand(req.Action, args)
|
||||||
} else {
|
} else {
|
||||||
if !f.ws.IsAgentReachable(id) {
|
|
||||||
writeJSON(w, map[string]interface{}{
|
|
||||||
"success": false,
|
|
||||||
"error": "agent not connected",
|
|
||||||
"agent_id": id,
|
|
||||||
"action": req.Action,
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
queued = !f.ws.isAgentConnected(id)
|
queued = !f.ws.isAgentConnected(id)
|
||||||
if err := f.ws.SendAgentCommand(id, req.Action, args); err != nil {
|
if err := f.ws.SendAgentCommand(id, req.Action, args); err != nil {
|
||||||
writeJSON(w, map[string]interface{}{
|
writeJSON(w, map[string]interface{}{
|
||||||
|
|||||||
@@ -105,22 +105,11 @@ func connectTestAgent(t *testing.T, hub *WSHub, agentID string) *websocket.Conn
|
|||||||
}
|
}
|
||||||
t.Cleanup(func() { _ = conn.Close() })
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
|
|
||||||
authPayload, _ := json.Marshal(map[string]interface{}{
|
authAgentConn(t, conn, map[string]interface{}{
|
||||||
"agent_id": agentID,
|
"agent_id": agentID,
|
||||||
"hostname": "test-host",
|
"hostname": "test-host",
|
||||||
"version": "1.0",
|
"version": "1.0",
|
||||||
})
|
})
|
||||||
if err := conn.WriteJSON(Message{Type: "auth", Payload: authPayload}); err != nil {
|
|
||||||
t.Fatalf("send auth: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var resp Message
|
|
||||||
if err := conn.ReadJSON(&resp); err != nil {
|
|
||||||
t.Fatalf("read auth_response: %v", err)
|
|
||||||
}
|
|
||||||
if resp.Type != "auth_response" {
|
|
||||||
t.Fatalf("expected auth_response, got %q", resp.Type)
|
|
||||||
}
|
|
||||||
|
|
||||||
deadline := time.Now().Add(2 * time.Second)
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
for time.Now().Before(deadline) {
|
for time.Now().Before(deadline) {
|
||||||
@@ -664,7 +653,7 @@ func TestFleetPostAgentCommandErrors(t *testing.T) {
|
|||||||
t.Run("agent not connected", func(t *testing.T) {
|
t.Run("agent not connected", func(t *testing.T) {
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
req := httptest.NewRequest(http.MethodPost, "/agents/offline-agent/command",
|
req := httptest.NewRequest(http.MethodPost, "/agents/offline-agent/command",
|
||||||
strings.NewReader(`{"action":"pause"}`))
|
strings.NewReader(`{"action":"get_log"}`))
|
||||||
fleetChiRoute(http.MethodPost, "/agents/{id}/command", fh.PostAgentCommand).ServeHTTP(rec, req)
|
fleetChiRoute(http.MethodPost, "/agents/{id}/command", fh.PostAgentCommand).ServeHTTP(rec, req)
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
t.Fatalf("expected 200, got %d body %s", rec.Code, rec.Body.String())
|
t.Fatalf("expected 200, got %d body %s", rec.Code, rec.Body.String())
|
||||||
@@ -711,13 +700,7 @@ func TestFleetPostAgentCommandSuccess(t *testing.T) {
|
|||||||
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
|
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
var cmd Message
|
cmd := readAgentWSMessage(t, conn, "command")
|
||||||
if err := conn.ReadJSON(&cmd); err != nil {
|
|
||||||
t.Fatalf("read command: %v", err)
|
|
||||||
}
|
|
||||||
if cmd.Type != "command" {
|
|
||||||
t.Fatalf("expected command message, got %q", cmd.Type)
|
|
||||||
}
|
|
||||||
var payload map[string]interface{}
|
var payload map[string]interface{}
|
||||||
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
|
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -739,10 +722,7 @@ func TestFleetPostAgentCommandBroadcastAll(t *testing.T) {
|
|||||||
t.Fatalf("status %d", rec.Code)
|
t.Fatalf("status %d", rec.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cmd Message
|
cmd := readAgentWSMessage(t, conn, "command")
|
||||||
if err := conn.ReadJSON(&cmd); err != nil {
|
|
||||||
t.Fatalf("read broadcast command: %v", err)
|
|
||||||
}
|
|
||||||
if cmd.Type != "command" {
|
if cmd.Type != "command" {
|
||||||
t.Fatalf("expected command, got %q", cmd.Type)
|
t.Fatalf("expected command, got %q", cmd.Type)
|
||||||
}
|
}
|
||||||
@@ -929,10 +909,7 @@ func TestFleetGetLogRefreshUsesTailConstant(t *testing.T) {
|
|||||||
t.Fatalf("status %d", rec.Code)
|
t.Fatalf("status %d", rec.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cmd Message
|
cmd := readAgentWSMessage(t, conn, "command")
|
||||||
if err := conn.ReadJSON(&cmd); err != nil {
|
|
||||||
t.Fatalf("read get_log command: %v", err)
|
|
||||||
}
|
|
||||||
var payload map[string]interface{}
|
var payload map[string]interface{}
|
||||||
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
|
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -287,21 +287,11 @@ func connectAgentViaRouter(t *testing.T, router http.Handler, agentID string) (*
|
|||||||
}
|
}
|
||||||
t.Cleanup(func() { _ = conn.Close() })
|
t.Cleanup(func() { _ = conn.Close() })
|
||||||
|
|
||||||
authPayload, _ := json.Marshal(map[string]interface{}{
|
authAgentConn(t, conn, map[string]interface{}{
|
||||||
"agent_id": agentID,
|
"agent_id": agentID,
|
||||||
"hostname": "integration-host",
|
"hostname": "integration-host",
|
||||||
"version": "1.0",
|
"version": "1.0",
|
||||||
})
|
})
|
||||||
if err := conn.WriteJSON(Message{Type: "auth", Payload: authPayload}); err != nil {
|
|
||||||
t.Fatalf("send auth: %v", err)
|
|
||||||
}
|
|
||||||
var resp Message
|
|
||||||
if err := conn.ReadJSON(&resp); err != nil {
|
|
||||||
t.Fatalf("read auth_response: %v", err)
|
|
||||||
}
|
|
||||||
if resp.Type != "auth_response" {
|
|
||||||
t.Fatalf("expected auth_response, got %q", resp.Type)
|
|
||||||
}
|
|
||||||
return conn, srv
|
return conn, srv
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -932,8 +922,7 @@ func TestIntegrationRouterWebSocketAgentConnectedCommand(t *testing.T) {
|
|||||||
func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
|
func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
|
||||||
router, wsHub, _, _ := newTestRouter(t)
|
router, wsHub, _, _ := newTestRouter(t)
|
||||||
agentID := "router-roundtrip-agent"
|
agentID := "router-roundtrip-agent"
|
||||||
const testAction = "exec"
|
const testAction = "pause"
|
||||||
const testCommand = "whoami"
|
|
||||||
const resultMessage = "integration round-trip ok"
|
const resultMessage = "integration round-trip ok"
|
||||||
|
|
||||||
agentConn, srv := connectAgentViaRouter(t, router, agentID)
|
agentConn, srv := connectAgentViaRouter(t, router, agentID)
|
||||||
@@ -989,26 +978,31 @@ func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
|
|||||||
agentCmdCh := make(chan agentCmdResult, 1)
|
agentCmdCh := make(chan agentCmdResult, 1)
|
||||||
go func() {
|
go func() {
|
||||||
_ = agentConn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
_ = agentConn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||||
var cmd Message
|
for {
|
||||||
if err := agentConn.ReadJSON(&cmd); err != nil {
|
var cmd Message
|
||||||
agentCmdCh <- agentCmdResult{err: err.Error()}
|
if err := agentConn.ReadJSON(&cmd); err != nil {
|
||||||
return
|
agentCmdCh <- agentCmdResult{err: err.Error()}
|
||||||
}
|
return
|
||||||
agentCmdCh <- agentCmdResult{cmd: cmd}
|
}
|
||||||
|
if cmd.Type != "command" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
agentCmdCh <- agentCmdResult{cmd: cmd}
|
||||||
|
|
||||||
cmdPayload, _ := json.Marshal(map[string]interface{}{
|
cmdPayload, _ := json.Marshal(map[string]interface{}{
|
||||||
"action": testAction,
|
"action": testAction,
|
||||||
"success": true,
|
"success": true,
|
||||||
"message": resultMessage,
|
"message": resultMessage,
|
||||||
})
|
})
|
||||||
if err := agentConn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}); err != nil {
|
if err := agentConn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}); err != nil {
|
||||||
agentCmdCh <- agentCmdResult{err: "send command_result: " + err.Error()}
|
agentCmdCh <- agentCmdResult{err: "send command_result: " + err.Error()}
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
cmdBody, _ := json.Marshal(map[string]string{
|
cmdBody, _ := json.Marshal(map[string]string{
|
||||||
"action": testAction,
|
"action": testAction,
|
||||||
"command": testCommand,
|
|
||||||
})
|
})
|
||||||
rec := serveAuthed(t, router, http.MethodPost, "/api/v1/agents/"+agentID+"/command", cmdBody)
|
rec := serveAuthed(t, router, http.MethodPost, "/api/v1/agents/"+agentID+"/command", cmdBody)
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
@@ -1040,9 +1034,6 @@ func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
|
|||||||
if payload["action"] != testAction {
|
if payload["action"] != testAction {
|
||||||
t.Errorf("agent command action: got %v, want %s", payload["action"], testAction)
|
t.Errorf("agent command action: got %v, want %s", payload["action"], testAction)
|
||||||
}
|
}
|
||||||
if payload["command"] != testCommand {
|
|
||||||
t.Errorf("agent command: got %v, want %s", payload["command"], testCommand)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
case <-time.After(5 * time.Second):
|
||||||
t.Fatal("timed out waiting for agent command")
|
t.Fatal("timed out waiting for agent command")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,6 +52,38 @@ func TestAuthResponseIncludesAdaptiveStrategy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAuthResponseOmitsAdaptiveStrategyWhenAIControlEnabled(t *testing.T) {
|
||||||
|
database, err := db.New(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = database.Close() })
|
||||||
|
|
||||||
|
hub := NewWSHub(database)
|
||||||
|
hub.SetFleetSecret("test-secret")
|
||||||
|
hub.SetAdaptiveEngine(strategy.NewAdaptiveEngine(database, true))
|
||||||
|
hub.SetServerPolicy(ServerPolicy{AIControlEnabled: true})
|
||||||
|
|
||||||
|
conn, _ := dialAgentWS(t, hub)
|
||||||
|
resp := authAgentConn(t, conn, map[string]interface{}{
|
||||||
|
"agent_id": "agent-ai-1", "fleet_secret": "test-secret",
|
||||||
|
"wallet": "4" + repeatChar('B', 94), "hostname": "win-docker", "platform": "windows", "version": "test",
|
||||||
|
})
|
||||||
|
if resp.Type != "auth_response" {
|
||||||
|
t.Fatalf("expected auth_response, got %q", resp.Type)
|
||||||
|
}
|
||||||
|
var payload map[string]interface{}
|
||||||
|
if err := json.Unmarshal(resp.Payload, &payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if payload["success"] != true {
|
||||||
|
t.Fatalf("auth failed: %v", payload["error"])
|
||||||
|
}
|
||||||
|
if _, ok := payload["adaptive_strategy"]; ok {
|
||||||
|
t.Fatal("adaptive_strategy must be omitted when ai_control_enabled is true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func repeatChar(c byte, n int) string {
|
func repeatChar(c byte, n int) string {
|
||||||
buf := make([]byte, n)
|
buf := make([]byte, n)
|
||||||
for i := range buf {
|
for i := range buf {
|
||||||
|
|||||||
@@ -52,6 +52,22 @@ func dialAgentWS(t *testing.T, hub *WSHub) (*websocket.Conn, string) {
|
|||||||
return conn, wsURL
|
return conn, wsURL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readAgentWSMessage(t *testing.T, conn *websocket.Conn, wantType string) Message {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
var msg Message
|
||||||
|
if err := conn.ReadJSON(&msg); err != nil {
|
||||||
|
t.Fatalf("read %s: %v", wantType, err)
|
||||||
|
}
|
||||||
|
if msg.Type == wantType {
|
||||||
|
return msg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("timed out waiting for %s", wantType)
|
||||||
|
return Message{}
|
||||||
|
}
|
||||||
|
|
||||||
func authAgentConn(t *testing.T, conn *websocket.Conn, payload map[string]interface{}) Message {
|
func authAgentConn(t *testing.T, conn *websocket.Conn, payload map[string]interface{}) Message {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, _ := json.Marshal(payload)
|
data, _ := json.Marshal(payload)
|
||||||
|
|||||||
196
server/web/src/pages/LotlTimelinePage.test.tsx
Normal file
196
server/web/src/pages/LotlTimelinePage.test.tsx
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
/**
|
||||||
|
* @vitest-environment happy-dom
|
||||||
|
*/
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { cleanup, render, screen, waitFor } from '@testing-library/react';
|
||||||
|
import userEvent from '@testing-library/user-event';
|
||||||
|
import { MemoryRouter, Route, Routes } from 'react-router-dom';
|
||||||
|
import LotlTimelinePage from './LotlTimelinePage';
|
||||||
|
import { mockAgent, mockServerConfig } from '../test/fixtures';
|
||||||
|
import { routerFuture } from '../routerFuture';
|
||||||
|
import { useWebSocket } from '../hooks/useWebSocket';
|
||||||
|
import { api } from '../api/client';
|
||||||
|
|
||||||
|
vi.mock('../hooks/useWebSocket', () => ({
|
||||||
|
useWebSocket: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('../components/HelpTip', () => ({
|
||||||
|
HelpTip: () => null,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const useWebSocketMock = vi.mocked(useWebSocket);
|
||||||
|
|
||||||
|
const agentA = mockAgent({
|
||||||
|
id: 'agent-a',
|
||||||
|
name: 'Alpha Node',
|
||||||
|
status: 'online',
|
||||||
|
lotl_tier: 'wsl',
|
||||||
|
lotl_attempts: [
|
||||||
|
{ tier: 'vuln_recon', ok: true, duration_ms: 400, phase: 'recon' },
|
||||||
|
{ tier: 'docker', ok: false, error: 'no daemon', duration_ms: 900, phase: 'deploy' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
function renderPage(initial = '/lotl-timeline') {
|
||||||
|
return render(
|
||||||
|
<MemoryRouter future={routerFuture} initialEntries={[initial]}>
|
||||||
|
<Routes>
|
||||||
|
<Route path="/lotl-timeline" element={<LotlTimelinePage />} />
|
||||||
|
</Routes>
|
||||||
|
</MemoryRouter>,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('LotlTimelinePage', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.spyOn(api, 'getConfig').mockResolvedValue(mockServerConfig());
|
||||||
|
vi.spyOn(api, 'getAIDecisions').mockResolvedValue([]);
|
||||||
|
vi.spyOn(api, 'getClearanceEvents').mockResolvedValue([]);
|
||||||
|
useWebSocketMock.mockReturnValue({
|
||||||
|
isConnected: true,
|
||||||
|
agents: [agentA],
|
||||||
|
recentShares: [],
|
||||||
|
fleetAlerts: [],
|
||||||
|
poolStatus: [],
|
||||||
|
aiActivity: [],
|
||||||
|
agentLogs: {},
|
||||||
|
commandResults: [],
|
||||||
|
latestMessage: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('renders LOTL Timeline heading and tier chain for online agent', async () => {
|
||||||
|
renderPage();
|
||||||
|
expect(await screen.findByRole('heading', { name: /LOTL Timeline/i })).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('ONION TIER CHAIN')).toBeInTheDocument();
|
||||||
|
expect(screen.getAllByText('Alpha Node').length).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(screen.getByText('Vuln Recon')).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows empty state when fleet has no agents', async () => {
|
||||||
|
useWebSocketMock.mockReturnValue({
|
||||||
|
isConnected: true,
|
||||||
|
agents: [],
|
||||||
|
recentShares: [],
|
||||||
|
fleetAlerts: [],
|
||||||
|
poolStatus: [],
|
||||||
|
aiActivity: [],
|
||||||
|
agentLogs: {},
|
||||||
|
commandResults: [],
|
||||||
|
latestMessage: null,
|
||||||
|
});
|
||||||
|
renderPage();
|
||||||
|
expect(
|
||||||
|
await screen.findByText(/No agents in fleet yet/i),
|
||||||
|
).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads AI decision panel when ai_control_enabled is true', async () => {
|
||||||
|
vi.mocked(api.getConfig).mockResolvedValue(
|
||||||
|
mockServerConfig({ server: { ai_control_enabled: true } }),
|
||||||
|
);
|
||||||
|
vi.mocked(api.getAIDecisions).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
agent_id: 'agent-a',
|
||||||
|
response: 'restart mining after docker failure',
|
||||||
|
commands_executed: 'restart_mining:ok',
|
||||||
|
ts: '2026-06-07T12:00:00Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
renderPage();
|
||||||
|
expect(await screen.findByText('LAST AI DECISION')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('restart mining after docker failure')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('restart_mining:ok')).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads court session decision with prosecutor/defender/judge roles', async () => {
|
||||||
|
vi.mocked(api.getConfig).mockResolvedValue(
|
||||||
|
mockServerConfig({ server: { ai_control_enabled: true } }),
|
||||||
|
);
|
||||||
|
vi.mocked(api.getAIDecisions).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: 2,
|
||||||
|
agent_id: 'agent-a',
|
||||||
|
response: 'Verdict: restart mining after tier exhaustion.',
|
||||||
|
commands_executed: 'restart_mining:ok',
|
||||||
|
court_session: true,
|
||||||
|
prosecutor_snippet: 'Failure atlas: docker 8/8 failed',
|
||||||
|
defender_snippet: 'Fleet phenotype from worker-07',
|
||||||
|
judge_verdict: 'restart mining after tier exhaustion.',
|
||||||
|
ts: '2026-06-07T12:05:00Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
renderPage();
|
||||||
|
expect(await screen.findByText('SINGULAR MACHINE COURT')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Prosecutor')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Defender')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Judge')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText(/Failure atlas: docker 8\/8 failed/i)).toBeInTheDocument();
|
||||||
|
expect(screen.getByText(/Fleet phenotype from worker-07/i)).toBeInTheDocument();
|
||||||
|
expect(screen.getByText(/restart mining after tier exhaustion/i)).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('selects agent from ?agent= query param', async () => {
|
||||||
|
const agentB = mockAgent({ id: 'agent-b', name: 'Beta Node', status: 'online' });
|
||||||
|
useWebSocketMock.mockReturnValue({
|
||||||
|
isConnected: true,
|
||||||
|
agents: [agentA, agentB],
|
||||||
|
recentShares: [],
|
||||||
|
fleetAlerts: [],
|
||||||
|
poolStatus: [],
|
||||||
|
aiActivity: [],
|
||||||
|
agentLogs: {},
|
||||||
|
commandResults: [],
|
||||||
|
latestMessage: null,
|
||||||
|
});
|
||||||
|
renderPage('/lotl-timeline?agent=agent-b');
|
||||||
|
const betaNodes = await screen.findAllByText('Beta Node');
|
||||||
|
expect(betaNodes.length).toBeGreaterThanOrEqual(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('renders clearance history when events exist', async () => {
|
||||||
|
vi.mocked(api.getClearanceEvents).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: 7,
|
||||||
|
agent_id: 'agent-a',
|
||||||
|
from_level: 1,
|
||||||
|
to_level: 2,
|
||||||
|
reason: 'spread lane needed',
|
||||||
|
source: 'ai_scheduler',
|
||||||
|
ts: '2026-06-07T11:00:00Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
renderPage();
|
||||||
|
expect(await screen.findByText('CLEARANCE HISTORY')).toBeInTheDocument();
|
||||||
|
expect(screen.getAllByText(/AI: L1 → L2/i).length).toBeGreaterThanOrEqual(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('switches selected agent from fleet overview', async () => {
|
||||||
|
const agentB = mockAgent({ id: 'agent-b', name: 'Beta Node', status: 'online' });
|
||||||
|
useWebSocketMock.mockReturnValue({
|
||||||
|
isConnected: true,
|
||||||
|
agents: [agentA, agentB],
|
||||||
|
recentShares: [],
|
||||||
|
fleetAlerts: [],
|
||||||
|
poolStatus: [],
|
||||||
|
aiActivity: [],
|
||||||
|
agentLogs: {},
|
||||||
|
commandResults: [],
|
||||||
|
latestMessage: null,
|
||||||
|
});
|
||||||
|
const user = userEvent.setup();
|
||||||
|
renderPage();
|
||||||
|
await screen.findAllByText('Alpha Node');
|
||||||
|
await user.click(screen.getByRole('button', { name: /Beta Node/i }));
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getAllByText('Beta Node').length).toBeGreaterThanOrEqual(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
135
tests/README.md
135
tests/README.md
@@ -56,6 +56,85 @@ The server **adaptive strategy engine** (`server/internal/strategy/`) learns fro
|
|||||||
|
|
||||||
Regression: `go test ./internal/strategy/... ./internal/api/ -run Adaptive` (server) and Vitest `AccessDepthPanel.test.tsx`.
|
Regression: `go test ./internal/strategy/... ./internal/api/ -run Adaptive` (server) and Vitest `AccessDepthPanel.test.tsx`.
|
||||||
|
|
||||||
|
## Phenotype cloning
|
||||||
|
|
||||||
|
When an agent reports a winning spread+mining path, the server upserts a **fleet phenotype** keyed by host fingerprint. Sibling agents receive `inherited_phenotype` on auth — tier order and spread lane clone without re-forge. Inherited phenotype **overrides** adaptive strategy on auth.
|
||||||
|
|
||||||
|
## Failure atlas
|
||||||
|
|
||||||
|
The failure atlas (`server/internal/atlas/`) records conditioned tier failures. After five failures under an active condition, it hard-skips subtree tiers, merges into `adaptive_strategy.skip_tiers`, and pushes `atlas_skips` on auth. LOTL Timeline marks tiers `skipped_by_atlas`.
|
||||||
|
|
||||||
|
## Court session
|
||||||
|
|
||||||
|
When AI Control is on and a host is stuck (zero hashrate + exhausted chain or all spread tiers failed), the scheduler runs a **Singular Machine Court**: Prosecutor (failure atlas + attempts), Defender (fleet phenotype), Judge (verdict + commands). Persisted with `court_session=true` for LOTL Timeline.
|
||||||
|
|
||||||
|
## Clearance L0–L4
|
||||||
|
|
||||||
|
Agents receive session clearance on auth (L0 stats → L4 forge). Fleet AI and remote actions enforce minimum levels. With `ai_auto_elevate_clearance`, stuck hosts auto-elevate to L4 so court-ordered commands can execute. Events broadcast as `clearance_elevated` on dashboard WS.
|
||||||
|
|
||||||
|
## Fleet AI Control
|
||||||
|
|
||||||
|
Calibrate → **Calibration Control** toggles `server.ai_control_enabled`. When **on**, the server **Fleet AI scheduler** (`server/internal/ai/`) polls connected agents on `ai_decision_interval_sec` (default 60s), builds snapshots from WS + DB state, calls a local OpenAI-compatible endpoint (`ai_endpoint`, default `http://127.0.0.1:11434/v1`), parses `commands[]` from the model response, and dispatches fleet actions (`restart_mining`, `discover_and_join`, `spread_now`, `agent_command`, etc.). Decisions are stored in SQLite `ai_decisions` and surfaced on **LOTL Timeline** when AI control is enabled.
|
||||||
|
|
||||||
|
**Precedence:** `ai_control_enabled: true` **replaces** adaptive strategy for tier-order decisions — auth omits `adaptive_strategy`, background rescoring no-ops, and `FleetAISnapshot` skips adaptive reasoning. Adaptive strategy resumes when AI control is turned off.
|
||||||
|
|
||||||
|
Operator settings: `ai_endpoint`, `ai_model`, `ai_no_context` (single-turn prompts), `ai_decision_interval_sec`. Refresh models: Calibrate **Refresh models** → `GET /api/v1/ai/models`. Audit trail: `GET /api/v1/ai/decisions?agent_id=`.
|
||||||
|
|
||||||
|
Agent side: hub sends `ai_snapshot_request` → agent replies `ai_snapshot` (`agent/client/ai_snapshot.go`); scheduler commands map to `ai_commands` handlers (`exec_shell`, `full_sys_check`, `restart_mining`, etc.). Per-agent Ollama autonomy (`ai_enabled` forge flag) remains separate — see [Agent logs](#agent-logs-not-a-missing-api).
|
||||||
|
|
||||||
|
### Fleet AI + LOTL Timeline quick-run
|
||||||
|
|
||||||
|
```bat
|
||||||
|
cd server && go test ./internal/ai/... ./internal/api/... -run "FleetAI|Scheduler|ParseCommands|AuthResponse.*Adaptive|AI" -count=1
|
||||||
|
cd agent && go test ./client/... -run "AI|HandleAI|AISnapshot|VulnLOTL" -count=1
|
||||||
|
cd server\web && npm run test -- --run src/pages/LotlTimelinePage.test.tsx src/pages/SettingsPage.test.tsx src/components/Lotl/LotlTierTimeline.test.tsx src/help/settingHelp.test.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fleet AI coverage map
|
||||||
|
|
||||||
|
| Feature | Test file(s) | Suite phase |
|
||||||
|
|---------|----------------|-------------|
|
||||||
|
| Command parser (JSON, tool-call, COMMAND: lines) | `server/internal/ai/commands_test.go` | 1 |
|
||||||
|
| OpenAI client (models list, decide) | `server/internal/ai/client_test.go` | 1 |
|
||||||
|
| Scheduler mock (1 agent, 1 cycle; disabled no-op) | `server/internal/ai/scheduler_test.go` | 1 |
|
||||||
|
| AI config / models / decisions API | `server/internal/api/fleet_ai_handler_test.go` | 1 |
|
||||||
|
| AI control precedence over adaptive (auth + snapshot) | `server/internal/api/strategy_auth_test.go`, `fleet_ai_handler_test.go` | 1 |
|
||||||
|
| Legacy Ollama decide/report API | `server/internal/api/ai_handler_test.go` | 1 |
|
||||||
|
| `ai_snapshot` JSON shape + stuck detection | `agent/client/ai_snapshot_test.go` | 2 |
|
||||||
|
| `ai_snapshot_request` WS dispatch | `agent/client/handlemessage_test.go` | 2 |
|
||||||
|
| `ai_commands` handlers + path traversal | `agent/client/ai_commands_test.go` | 2 |
|
||||||
|
| Calibrate AI Control toggle + models refresh | `server/web/src/pages/SettingsPage.test.tsx` | 4 |
|
||||||
|
| LOTL Timeline page (tier chain + AI decision panel) | `server/web/src/pages/LotlTimelinePage.test.tsx` | 4 |
|
||||||
|
| LOTL tier timeline component | `server/web/src/components/Lotl/LotlTierTimeline.test.tsx` | 4 |
|
||||||
|
| Calibrate help keys (`calibration_ai_control`) | `server/web/src/help/settingHelp.test.ts`, `docAnchors.test.ts` | 4 |
|
||||||
|
| Singular Machine Court prompts | `server/internal/ai/court_prompt_test.go` | 1 |
|
||||||
|
|
||||||
|
### Fleet intelligence (2026-06-07 — phenotype, atlas, court, clearance)
|
||||||
|
|
||||||
|
| Feature | Test file(s) | Suite phase |
|
||||||
|
|---------|----------------|-------------|
|
||||||
|
| Fleet phenotype store + peak hashrate | `server/internal/strategy/phenotype_test.go`, `server/internal/db/phenotype_test.go` | 1 |
|
||||||
|
| Phenotype publish + sibling inheritance API | `server/internal/api/phenotype_test.go` | 1 |
|
||||||
|
| Agent auth phenotype policy | `agent/client/phenotype_policy_test.go` | 2 |
|
||||||
|
| Failure atlas subtree skips | `server/internal/atlas/failure_atlas_test.go` | 1 |
|
||||||
|
| Clearance L0–L4 command gating | `server/internal/clearance/clearance_test.go` | 1 |
|
||||||
|
| AI scheduler clearance elevation | `server/internal/ai/scheduler_test.go` | 1 |
|
||||||
|
| Clearance helpers + timeline history | `server/web/src/help/clearance.test.ts`, `server/web/src/pages/LotlTimelinePage.test.tsx` | 4 |
|
||||||
|
| Phenotype cloned-from + clearance badge UI | `server/web/src/components/Lotl/LotlTierTimeline.test.tsx`, `server/web/src/components/Fleet/AccessDepthPanel.test.tsx` | 4 |
|
||||||
|
|
||||||
|
```bat
|
||||||
|
cd server && go test ./internal/strategy/... ./internal/db/... ./internal/api/... ./internal/atlas/... ./internal/clearance/... ./internal/ai/... -run "Phenotype|Atlas|Court|Clearance" -count=1
|
||||||
|
cd agent && go test ./client/... -run Phenotype -count=1
|
||||||
|
cd server\web && npm run test -- --run src/help/clearance.test.ts src/components/Fleet/AccessDepthPanel.test.tsx src/components/Lotl/LotlTierTimeline.test.tsx src/pages/LotlTimelinePage.test.tsx
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fleet AI gaps
|
||||||
|
|
||||||
|
- **Live Ollama / vLLM inference** — scheduler uses `DecideFunc` inject in unit tests; no CI container with a real model.
|
||||||
|
- **Full scheduler E2E** — one mocked `Tick()` cycle covered; no multi-agent parallel decision race test.
|
||||||
|
- **Court session UI** — Go + Vitest cover prosecutor/defender/judge in `LotlTimelinePage.test.tsx`; no Playwright path yet.
|
||||||
|
- **Real `full_sys_check` syscheck bundle** — handler test stubs `CollectFullSysCheck`; live subnet scan / `systeminfo` not exercised in CI.
|
||||||
|
|
||||||
## LOTL architecture (triple onion)
|
## LOTL architecture (triple onion)
|
||||||
|
|
||||||
The **triple onion** chains three phases on every agent connect (when enabled): **recon → deploy → mining**. Policy gates (`patch_first`, `skip_mining_on_high_risk`) can defer deploy or mining when `vuln_findings` exceed thresholds.
|
The **triple onion** chains three phases on every agent connect (when enabled): **recon → deploy → mining**. Policy gates (`patch_first`, `skip_mining_on_high_risk`) can defer deploy or mining when `vuln_findings` exceed thresholds.
|
||||||
@@ -103,6 +182,30 @@ flowchart TB
|
|||||||
deploy -->|all lanes fail| mining
|
deploy -->|all lanes fail| mining
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Phenotype inherit and failure-atlas skip branches (adaptive / auth path):
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
subgraph auth["Agent auth"]
|
||||||
|
fp[fingerprint match]
|
||||||
|
pheno{winning phenotype?}
|
||||||
|
inherit[inherited_phenotype tier_order + spread_lane]
|
||||||
|
adaptive[adaptive_strategy tier_order]
|
||||||
|
atlasRec[atlas RecordFailure from stats]
|
||||||
|
atlasSkip[atlas_skips hard subtree]
|
||||||
|
merge[MergeSkipsIntoStrategy skip_tiers]
|
||||||
|
end
|
||||||
|
|
||||||
|
fp --> pheno
|
||||||
|
pheno -->|yes| inherit
|
||||||
|
pheno -->|no| adaptive
|
||||||
|
atlasRec --> atlasSkip
|
||||||
|
adaptive --> merge
|
||||||
|
atlasSkip --> merge
|
||||||
|
inherit --> agentPolicy[agent tier policy]
|
||||||
|
merge --> agentPolicy
|
||||||
|
```
|
||||||
|
|
||||||
Sequential tier attempts within each phase (mining chain shown; spread/deploy lanes behave the same way):
|
Sequential tier attempts within each phase (mining chain shown; spread/deploy lanes behave the same way):
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@@ -256,7 +359,8 @@ All Go packages under `server/` and `agent/` are picked up automatically by `go
|
|||||||
|------|-------------------|------|-------|
|
|------|-------------------|------|-------|
|
||||||
| `TestIntegrationRouterCommandFullRoundTrip` | API `POST /command` → agent WS → `command_result` → dashboard WS | `server/internal/api/integration_test.go` | 1 |
|
| `TestIntegrationRouterCommandFullRoundTrip` | API `POST /command` → agent WS → `command_result` → dashboard WS | `server/internal/api/integration_test.go` | 1 |
|
||||||
| `TestAllowAgentWSUpgradeRateLimit` | 31st `/ws/agent` upgrade from same IP within 1 min rejected; empty IP allowed | `server/internal/api/agent_ws_limiter_test.go` | 1 |
|
| `TestAllowAgentWSUpgradeRateLimit` | 31st `/ws/agent` upgrade from same IP within 1 min rejected; empty IP allowed | `server/internal/api/agent_ws_limiter_test.go` | 1 |
|
||||||
| Crucible exec E2E | Online stub agent; **whoami**, terminal **echo**, and **LOTL tier badge** on `/crucible` | `server/web/e2e/crucible-command.spec.ts` | 8 |
|
| Crucible exec E2E | Online stub agent; **whoami** and terminal **echo** on `/crucible` | `server/web/e2e/crucible-command.spec.ts` | 8 |
|
||||||
|
| Crucible LOTL E2E | Stub **LOTL tier badge** on Crucible + **Onion timeline** tier chain | `server/web/e2e/crucible-lotl.spec.ts` | 8 |
|
||||||
| `TestPathForgeRootPathOutsideAllowedRoots` | PathForge `root_path` outside allowlist → HTTP 400, `Placed=0` | `server/internal/builder/pathforge_test.go` | 1 |
|
| `TestPathForgeRootPathOutsideAllowedRoots` | PathForge `root_path` outside allowlist → HTTP 400, `Placed=0` | `server/internal/builder/pathforge_test.go` | 1 |
|
||||||
| `TestUploadCommandRejectsPathTraversal` | Agent `upload` blocks `../../` via `ResolveRemotePath` | `agent/client/client_upload_test.go` | 2 |
|
| `TestUploadCommandRejectsPathTraversal` | Agent `upload` blocks `../../` via `ResolveRemotePath` | `agent/client/client_upload_test.go` | 2 |
|
||||||
|
|
||||||
@@ -281,9 +385,11 @@ Run Crucible P0 E2E only (needs a live server on `AETHERFORGE_URL`, default `:89
|
|||||||
set AETHERFORGE_E2E_USER=testuser
|
set AETHERFORGE_E2E_USER=testuser
|
||||||
set AETHERFORGE_E2E_PASS=testpass
|
set AETHERFORGE_E2E_PASS=testpass
|
||||||
set AETHERFORGE_URL=http://127.0.0.1:8989
|
set AETHERFORGE_URL=http://127.0.0.1:8989
|
||||||
cd server\web && npx playwright test e2e/crucible-command.spec.ts
|
cd server\web && npx playwright test e2e/crucible-command.spec.ts e2e/crucible-lotl.spec.ts
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`e2e/fixtures.ts` exports `waitForServerHealth()` — polls `/api/v1/health` for up to 30s (used by live-server specs to avoid flakes on cold start). Phase 8 sets `AETHERFORGE_FLEET_SECRET` from `data/config.json` so stub agents authenticate without scraping `/api/v1/config`.
|
||||||
|
|
||||||
`e2e/remote-actions.spec.ts` mocks the dashboard WebSocket `init` payload (Crucible prefers live WS fleet data over REST). Playwright HTTP `page.route` alone cannot intercept WebSockets in this toolchain version. Asserts mining Pause/Resume in `.cop-mining` and bulk Pause in `.fleet-bulk-bar` when only an offline agent is selected.
|
`e2e/remote-actions.spec.ts` mocks the dashboard WebSocket `init` payload (Crucible prefers live WS fleet data over REST). Playwright HTTP `page.route` alone cannot intercept WebSockets in this toolchain version. Asserts mining Pause/Resume in `.cop-mining` and bulk Pause in `.fleet-bulk-bar` when only an offline agent is selected.
|
||||||
|
|
||||||
Run remote-actions only (no stub agent; mocks offline fleet via WS):
|
Run remote-actions only (no stub agent; mocks offline fleet via WS):
|
||||||
@@ -302,7 +408,9 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
|
|||||||
| P0 command round-trip (integration) | `server/internal/api/integration_test.go` | 1 |
|
| P0 command round-trip (integration) | `server/internal/api/integration_test.go` | 1 |
|
||||||
| P0 agent WS rate limit | `server/internal/api/agent_ws_limiter_test.go` | 1 |
|
| P0 agent WS rate limit | `server/internal/api/agent_ws_limiter_test.go` | 1 |
|
||||||
| P0 PathForge root rejection | `server/internal/builder/pathforge_test.go` | 1 |
|
| P0 PathForge root rejection | `server/internal/builder/pathforge_test.go` | 1 |
|
||||||
| P0 Crucible exec + LOTL badge E2E | `server/web/e2e/crucible-command.spec.ts` | 8 |
|
| P0 Crucible exec E2E | `server/web/e2e/crucible-command.spec.ts` | 8 |
|
||||||
|
| P0 Crucible LOTL badge + Onion timeline E2E | `server/web/e2e/crucible-lotl.spec.ts` | 8 |
|
||||||
|
| Calibrate AI Control toggle E2E | `server/web/e2e/pages.spec.ts` (Logic gates / AI Control smoke) | 8 |
|
||||||
| P0 upload path traversal (P1 download) | `agent/client/client_upload_test.go` | 2 |
|
| P0 upload path traversal (P1 download) | `agent/client/client_upload_test.go` | 2 |
|
||||||
| Cascading fallback chain | `agent/miner/fallback_chain_test.go` | 2 |
|
| Cascading fallback chain | `agent/miner/fallback_chain_test.go` | 2 |
|
||||||
| Container mining / execution mode | `agent/miner/execution_test.go` | 2 |
|
| Container mining / execution mode | `agent/miner/execution_test.go` | 2 |
|
||||||
@@ -348,6 +456,19 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
|
|||||||
| Fleet bulk actions hook | `server/web/src/hooks/useFleetBulkActions.test.ts` | 4 |
|
| Fleet bulk actions hook | `server/web/src/hooks/useFleetBulkActions.test.ts` | 4 |
|
||||||
| War Room LOTL/join-lane telemetry | `server/web/src/help/warRoomTelemetry.test.ts` | 4 |
|
| War Room LOTL/join-lane telemetry | `server/web/src/help/warRoomTelemetry.test.ts` | 4 |
|
||||||
| Spread template export panel | `server/web/src/help/spreadTemplateExport.test.ts`, `SpreadTemplateExportPanel.tsx` | 4 |
|
| Spread template export panel | `server/web/src/help/spreadTemplateExport.test.ts`, `SpreadTemplateExportPanel.tsx` | 4 |
|
||||||
|
|
||||||
|
### Fleet intelligence (2026-06-07 parallel agents)
|
||||||
|
|
||||||
|
| Feature | Test file(s) | Suite phase |
|
||||||
|
|---------|----------------|-------------|
|
||||||
|
| Phenotype publish + sibling inherit | `server/internal/api/phenotype_test.go`, `server/internal/db/phenotype_test.go`, `agent/client/phenotype_policy_test.go` | 1 / 2 |
|
||||||
|
| Failure atlas subtree skip | `server/internal/atlas/failure_atlas_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
|
||||||
|
| Singular Machine Court | `server/internal/ai/court_prompt_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
|
||||||
|
| Clearance L0–L4 enforcement | `server/internal/clearance/clearance_test.go`, `server/internal/api/fleet_intelligence_test.go` | 1 |
|
||||||
|
| Access Depth phenotype + clearance badge | `AccessDepthPanel.test.tsx`, `clearance.test.ts` | 4 |
|
||||||
|
| LOTL Timeline atlas skip + cloned-from | `lotlTimeline.test.ts`, `LotlTierTimeline.test.tsx` | 4 |
|
||||||
|
| Court decision UI | `LotlTimelinePage.test.tsx` | 4 |
|
||||||
|
| AI snapshot phenotype/atlas/clearance | `agent/client/ai_snapshot_test.go` | 2 |
|
||||||
| Service graph summary UI | `CrucibleExpandedOps.test.tsx` (mocked `ServiceGraphSummary`) | 4 |
|
| Service graph summary UI | `CrucibleExpandedOps.test.tsx` (mocked `ServiceGraphSummary`) | 4 |
|
||||||
| `vuln_findings` / `join_lane` WS stats merge | `applyStatsUpdate.test.ts`, `wsStatsCoalesce.test.ts` | 4 |
|
| `vuln_findings` / `join_lane` WS stats merge | `applyStatsUpdate.test.ts`, `wsStatsCoalesce.test.ts` | 4 |
|
||||||
| Emberwake `join_lane` funnel tag | `ReconBadges.test.tsx` (`JoinLaneBadge`), `WarRoomFunnelBoard.tsx` | 4 |
|
| Emberwake `join_lane` funnel tag | `ReconBadges.test.tsx` (`JoinLaneBadge`), `WarRoomFunnelBoard.tsx` | 4 |
|
||||||
@@ -369,6 +490,12 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
|
|||||||
| `TestNormalizeLotlTiers*` / `TestTryLotlTier*` | Spread onion tier normalization + unix stub tiers | `agent/deploy/lotl_tiers_test.go`, `lotl_onion_stub_test.go` | 2 |
|
| `TestNormalizeLotlTiers*` / `TestTryLotlTier*` | Spread onion tier normalization + unix stub tiers | `agent/deploy/lotl_tiers_test.go`, `lotl_onion_stub_test.go` | 2 |
|
||||||
| `TestStagingRejectsPathTraversal*` / `TestVerifyFileSHA256*` | BITS/curl/certutil staging path hygiene + hash verify | `agent/deploy/staging_test.go` | 2 |
|
| `TestStagingRejectsPathTraversal*` / `TestVerifyFileSHA256*` | BITS/curl/certutil staging path hygiene + hash verify | `agent/deploy/staging_test.go` | 2 |
|
||||||
| `TestValidateUNCSpreadPath*` / `TestSMBUNCSvcName*` | SMB sc.exe spread helpers | `agent/deploy/smb_unc_spread_test.go` | 2 |
|
| `TestValidateUNCSpreadPath*` / `TestSMBUNCSvcName*` | SMB sc.exe spread helpers | `agent/deploy/smb_unc_spread_test.go` | 2 |
|
||||||
|
| `TestDoPeer*` / do_peer staging | DoSvc shadow cache handoff — hash verify + launch | `agent/deploy/do_peer_staging_test.go` | 2 |
|
||||||
|
| `TestDNS*` / dns_txt staging | DNS TXT shard assembly + SHA256 verify | `agent/deploy/dns_txt_staging_test.go` | 2 |
|
||||||
|
| `TestWebRTCMesh*` | WebRTC mesh manifest receive (mock channel) | `agent/deploy/webrtc_mesh_test.go` | 2 |
|
||||||
|
| `TestWSUSCachePeer*` | WSUS cache cousin staging beside SoftwareDistribution | `agent/deploy/wsus_cache_peer_staging_test.go` | 2 |
|
||||||
|
| Deploy plan spread lanes | do_peer / dns_txt / webrtc_mesh / wsus_cache_peer signed plans | `server/internal/api/deploy_plan_test.go`, `agent/deploy/discover_join_test.go`, `server/internal/api/service_deploy_test.go` | 1 / 2 |
|
||||||
|
| Join lane labels (do_peer, dns_txt, webrtc, wsus) | Crucible/Emberwake badge copy | `server/web/src/help/reconRisk.test.ts`, `ReconBadges.test.tsx` | 4 |
|
||||||
| `TestMiningStatusRelayCoalescedToStatsBatch` | `mining_hashrate`, `lotl_tier`, `lotl_attempts` in stats_batch | `server/internal/api/websocket_test.go` | 1 |
|
| `TestMiningStatusRelayCoalescedToStatsBatch` | `mining_hashrate`, `lotl_tier`, `lotl_attempts` in stats_batch | `server/internal/api/websocket_test.go` | 1 |
|
||||||
| `TestStatsBatchCoalescesSameAgent` | Same-agent coalesce preserves LOTL fields | `server/internal/api/websocket_test.go` | 1 |
|
| `TestStatsBatchCoalescesSameAgent` | Same-agent coalesce preserves LOTL fields | `server/internal/api/websocket_test.go` | 1 |
|
||||||
| `TestAgentLotlFieldsJSONRoundTrip` | Agent model JSON exposes tier telemetry | `server/internal/models/agent_test.go` | 1 |
|
| `TestAgentLotlFieldsJSONRoundTrip` | Agent model JSON exposes tier telemetry | `server/internal/models/agent_test.go` | 1 |
|
||||||
@@ -377,7 +504,7 @@ cd server\web && npx playwright test e2e/remote-actions.spec.ts
|
|||||||
| `LotlTierBadge` / `LotlAttemptsList` | Crucible tier badge + attempt list UI | `server/web/src/components/Fleet/LotlTierBadge.test.tsx` | 4 |
|
| `LotlTierBadge` / `LotlAttemptsList` | Crucible tier badge + attempt list UI | `server/web/src/components/Fleet/LotlTierBadge.test.tsx` | 4 |
|
||||||
| `WebSocketProvider` stats_batch LOTL | Dashboard WS applies tier fields | `server/web/src/context/WebSocketProvider.test.tsx` | 4 |
|
| `WebSocketProvider` stats_batch LOTL | Dashboard WS applies tier fields | `server/web/src/context/WebSocketProvider.test.tsx` | 4 |
|
||||||
| Forge LOTL Onion preset UI | `applyOperationMode('lotl_onion')` flags | `server/web/src/help/forgeOperationModes.test.ts` | 4 |
|
| Forge LOTL Onion preset UI | `applyOperationMode('lotl_onion')` flags | `server/web/src/help/forgeOperationModes.test.ts` | 4 |
|
||||||
| LOTL onion tier docs | Ten-tier spread chain constants | `server/web/src/help/lotlOnionTiers.test.ts` | 4 |
|
| LOTL onion tier docs | 14-tier spread chain constants (sync with `DefaultLotlOnionTiers`) | `server/web/src/help/lotlOnionTiers.test.ts`, `agent/deploy/lotl_tiers_test.go`, `server/internal/builder/lotl_onion_test.go` | 2 / 4 |
|
||||||
| Fleet health bulk pause/resume | Bulk command framing + toolbar wiring | `server/internal/api/fleet_handler_test.go`, `components.test.tsx` | 1 / 4 |
|
| Fleet health bulk pause/resume | Bulk command framing + toolbar wiring | `server/internal/api/fleet_handler_test.go`, `components.test.tsx` | 1 / 4 |
|
||||||
|
|
||||||
Run LOTL Go tests quickly:
|
Run LOTL Go tests quickly:
|
||||||
|
|||||||
Reference in New Issue
Block a user