Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.
This commit is contained in:
AetherForge
326
agent/miner/fallback_chain_test.go
Normal file
326
agent/miner/fallback_chain_test.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package miner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"crypto-miner-agent/config"
|
||||
)
|
||||
|
||||
func TestDefaultFallbackChainAutoWithDocker(t *testing.T) {
|
||||
SetRuntimeDetector(func() ContainerRuntimeInfo {
|
||||
return ContainerRuntimeInfo{Available: true, CLI: "docker"}
|
||||
})
|
||||
defer SetRuntimeDetector(nil)
|
||||
SetWSLDetector(func() WSLRuntimeInfo { return WSLRuntimeInfo{} })
|
||||
defer SetWSLDetector(nil)
|
||||
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{
|
||||
MinerExecution: ExecutionAuto,
|
||||
PoolHost: "pool.example.com",
|
||||
},
|
||||
}, ContainerRuntimeInfo{Available: true, CLI: "docker"})
|
||||
|
||||
want := []MiningMethod{MethodContainer, MethodInProcess, MethodStratumDirect}
|
||||
if len(chain) < len(want) {
|
||||
t.Fatalf("chain=%v want at least %v", chain, want)
|
||||
}
|
||||
for i := range want {
|
||||
if chain[i] != want[i] {
|
||||
t.Fatalf("chain[%d]=%q want %q full=%v", i, chain[i], want[i], chain)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFallbackChainAutoWithoutRuntime(t *testing.T) {
|
||||
SetWSLDetector(func() WSLRuntimeInfo { return WSLRuntimeInfo{} })
|
||||
defer SetWSLDetector(nil)
|
||||
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionAuto, PoolHost: "p"},
|
||||
}, ContainerRuntimeInfo{})
|
||||
|
||||
if chain[0] != MethodInProcess {
|
||||
t.Fatalf("got %v want inprocess first", chain)
|
||||
}
|
||||
// Windows LOTL probe tiers trail stratum_direct in the legacy chain.
|
||||
stratumIdx := -1
|
||||
for i, m := range chain {
|
||||
if m == MethodStratumDirect {
|
||||
stratumIdx = i
|
||||
}
|
||||
}
|
||||
if stratumIdx < 0 {
|
||||
t.Fatalf("got %v want stratum_direct present", chain)
|
||||
}
|
||||
if runtime.GOOS != "windows" && chain[len(chain)-1] != MethodStratumDirect {
|
||||
t.Fatalf("got %v want stratum last", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFallbackChainInProcessSkipsContainer(t *testing.T) {
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{
|
||||
MinerExecution: ExecutionInProcess,
|
||||
GPUEnabled: true,
|
||||
RVNWallet: "wallet",
|
||||
PoolHost: "p",
|
||||
},
|
||||
}, ContainerRuntimeInfo{Available: true, CLI: "docker"})
|
||||
|
||||
for _, m := range chain {
|
||||
if m == MethodContainer {
|
||||
t.Fatal("inprocess mode must skip container")
|
||||
}
|
||||
}
|
||||
if chain[0] != MethodInProcess {
|
||||
t.Fatalf("got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFallbackChainGPUIncludedWhenConfigured(t *testing.T) {
|
||||
SetWSLDetector(func() WSLRuntimeInfo { return WSLRuntimeInfo{} })
|
||||
defer SetWSLDetector(nil)
|
||||
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{
|
||||
MinerExecution: ExecutionAuto,
|
||||
GPUEnabled: true,
|
||||
RVNWallet: "wallet",
|
||||
},
|
||||
}, ContainerRuntimeInfo{Available: true, CLI: "docker"})
|
||||
|
||||
foundGPU := false
|
||||
for _, m := range chain {
|
||||
if m == MethodGPUSubprocess {
|
||||
foundGPU = true
|
||||
}
|
||||
}
|
||||
if !foundGPU {
|
||||
t.Fatalf("expected gpu in chain, got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryChainOrderAndSkipMissingRuntime(t *testing.T) {
|
||||
var started []MiningMethod
|
||||
hooks := ChainHooks{
|
||||
StartContainer: func() error {
|
||||
started = append(started, MethodContainer)
|
||||
return errors.New("container start blocked")
|
||||
},
|
||||
StartInProcess: func() error {
|
||||
started = append(started, MethodInProcess)
|
||||
return nil
|
||||
},
|
||||
StartGPU: func() error { return nil },
|
||||
IsGPUSupported: func() bool { return false },
|
||||
}
|
||||
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{
|
||||
MinerExecution: ExecutionAuto,
|
||||
PoolHost: "p",
|
||||
},
|
||||
}, hooks, nil)
|
||||
ctrl.runtime = ContainerRuntimeInfo{Available: true, CLI: "docker"}
|
||||
ctrl.chain = DefaultFallbackChain(ctrl.cfg, ctrl.runtime)
|
||||
|
||||
method, err := ctrl.TryChain(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("TryChain: %v", err)
|
||||
}
|
||||
if method != MethodInProcess {
|
||||
t.Fatalf("active=%q want inprocess", method)
|
||||
}
|
||||
if len(started) != 2 || started[0] != MethodContainer || started[1] != MethodInProcess {
|
||||
t.Fatalf("start order=%v", started)
|
||||
}
|
||||
if len(ctrl.Status().FailedMethods) != 1 || ctrl.Status().FailedMethods[0].Method != MethodContainer {
|
||||
t.Fatalf("failures=%v", ctrl.Status().FailedMethods)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryChainExhausted(t *testing.T) {
|
||||
hooks := ChainHooks{
|
||||
StartContainer: func() error { return errors.New("no container") },
|
||||
StartInProcess: func() error { return errors.New("no inprocess") },
|
||||
}
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionInProcess},
|
||||
}, hooks, nil)
|
||||
ctrl.chain = []MiningMethod{MethodInProcess}
|
||||
|
||||
_, err := ctrl.TryChain(context.Background())
|
||||
if !errors.Is(err, ErrChainExhausted) && err.Error() != "no inprocess" {
|
||||
t.Fatalf("got err=%v", err)
|
||||
}
|
||||
if !ctrl.Status().ChainExhausted {
|
||||
t.Fatal("expected chain exhausted flag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvancePrimaryFromContainer(t *testing.T) {
|
||||
var inprocessStarted bool
|
||||
hooks := ChainHooks{
|
||||
StartInProcess: func() error {
|
||||
inprocessStarted = true
|
||||
return nil
|
||||
},
|
||||
StopContainer: func() {},
|
||||
}
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionAuto},
|
||||
}, hooks, nil)
|
||||
ctrl.chain = []MiningMethod{MethodContainer, MethodInProcess}
|
||||
ctrl.activePrimary = MethodContainer
|
||||
|
||||
ctrl.AdvancePrimary("container exited")
|
||||
if !inprocessStarted {
|
||||
t.Fatal("expected inprocess start after container failure")
|
||||
}
|
||||
if ctrl.Status().ActiveMethod != MethodInProcess {
|
||||
t.Fatalf("active=%q", ctrl.Status().ActiveMethod)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultChainCooldownConstant(t *testing.T) {
|
||||
if DefaultChainCooldown != 30*time.Second {
|
||||
t.Fatalf("DefaultChainCooldown = %v want 30s", DefaultChainCooldown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFallbackChainForcedContainerWithoutRuntime(t *testing.T) {
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{
|
||||
MinerExecution: ExecutionContainer,
|
||||
PoolHost: "p",
|
||||
},
|
||||
}, ContainerRuntimeInfo{})
|
||||
|
||||
for _, m := range chain {
|
||||
if m == MethodContainer {
|
||||
t.Fatal("container mode without runtime must skip container method")
|
||||
}
|
||||
}
|
||||
if chain[0] != MethodInProcess {
|
||||
t.Fatalf("got %v", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFallbackChainDockerLoadBeforeContainer(t *testing.T) {
|
||||
t.Setenv("AETHERFORGE_DOCKER_IMAGE_TAR", "")
|
||||
SetImageTarFetcher(func(cfg config.RuntimeConfig) (string, error) {
|
||||
return "/policy/worker.tar", nil
|
||||
})
|
||||
defer SetImageTarFetcher(nil)
|
||||
SetRuntimeDetector(func() ContainerRuntimeInfo {
|
||||
return ContainerRuntimeInfo{Available: true, CLI: "docker"}
|
||||
})
|
||||
defer SetRuntimeDetector(nil)
|
||||
SetWSLDetector(func() WSLRuntimeInfo { return WSLRuntimeInfo{} })
|
||||
defer SetWSLDetector(nil)
|
||||
|
||||
chain := DefaultFallbackChain(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionAuto, PoolHost: "p"},
|
||||
}, ContainerRuntimeInfo{Available: true, CLI: "docker"})
|
||||
|
||||
if chain[0] != MethodDockerLoad {
|
||||
t.Fatalf("chain=%v want docker_load first when tar policy set", chain)
|
||||
}
|
||||
if chain[1] != MethodContainer {
|
||||
t.Fatalf("chain=%v want container second", chain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryChainDockerLoadFailsReportsAttempt(t *testing.T) {
|
||||
SetImageTarFetcher(func(cfg config.RuntimeConfig) (string, error) {
|
||||
return "/tmp/worker.tar", nil
|
||||
})
|
||||
defer SetImageTarFetcher(nil)
|
||||
|
||||
var failed []MiningMethod
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionAuto, PoolHost: "p"},
|
||||
}, ChainHooks{
|
||||
StartDockerLoad: func() error { failed = append(failed, MethodDockerLoad); return errors.New("docker missing") },
|
||||
StartInProcess: func() error { return nil },
|
||||
}, nil)
|
||||
ctrl.runtime = ContainerRuntimeInfo{Available: true, CLI: "docker"}
|
||||
ctrl.chain = []MiningMethod{MethodDockerLoad, MethodInProcess}
|
||||
|
||||
method, err := ctrl.TryChain(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("TryChain: %v", err)
|
||||
}
|
||||
if method != MethodInProcess {
|
||||
t.Fatalf("active=%q want inprocess", method)
|
||||
}
|
||||
if len(ctrl.Status().FailedMethods) != 1 || ctrl.Status().FailedMethods[0].Method != MethodDockerLoad {
|
||||
t.Fatalf("failures=%v", ctrl.Status().FailedMethods)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryChainRunTierHooksPopulatesLOTLFields(t *testing.T) {
|
||||
hooks := ChainHooks{
|
||||
StartInProcess: func() error { return nil },
|
||||
RunTierProbes: func() TierReport {
|
||||
return TierReport{
|
||||
Attempts: []TierAttempt{{
|
||||
Tier: TierWebView2Probe,
|
||||
OK: true,
|
||||
Wallet: "same-wallet",
|
||||
}},
|
||||
WebGPUReady: true,
|
||||
}
|
||||
},
|
||||
RunTierChain: func() (LOTLTier, error) {
|
||||
return TierCPUInprocess, nil
|
||||
},
|
||||
}
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionInProcess},
|
||||
}, hooks, nil)
|
||||
ctrl.chain = []MiningMethod{MethodInProcess}
|
||||
|
||||
if _, err := ctrl.TryChain(context.Background()); err != nil {
|
||||
t.Fatalf("TryChain: %v", err)
|
||||
}
|
||||
st := ctrl.Status()
|
||||
if st.LOTLTier != TierCPUInprocess {
|
||||
t.Fatalf("lotl_tier=%q want cpu_inprocess", st.LOTLTier)
|
||||
}
|
||||
if len(st.LOTLAttempts) != 1 || st.LOTLAttempts[0].Tier != TierWebView2Probe {
|
||||
t.Fatalf("lotl_attempts=%v", st.LOTLAttempts)
|
||||
}
|
||||
if !st.WebGPUReady {
|
||||
t.Fatal("expected webgpu_ready from tier probes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryChainRespectsCooldown(t *testing.T) {
|
||||
attempts := 0
|
||||
hooks := ChainHooks{
|
||||
StartInProcess: func() error {
|
||||
attempts++
|
||||
return nil
|
||||
},
|
||||
}
|
||||
ctrl := NewChainController(config.RuntimeConfig{
|
||||
BuiltinConfig: config.BuiltinConfig{MinerExecution: ExecutionInProcess},
|
||||
}, hooks, nil)
|
||||
ctrl.chain = []MiningMethod{MethodInProcess}
|
||||
|
||||
if _, err := ctrl.TryChain(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ctrl.TryChain(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("attempts=%d want 1 (cooldown)", attempts)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user