Add mining self-surgery for on-host recovery when AI control detects stalls.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled

When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
This commit is contained in:
AetherForge
2026-06-07 09:27:26 -07:00
parent d605ef4adb
commit fac324ff80
18 changed files with 1166 additions and 0 deletions

View File

@@ -0,0 +1,107 @@
package api
import (
"encoding/json"
"crypto-miner-server/internal/epidemiology"
"crypto-miner-server/internal/miningsurgery"
)
func (h *WSHub) miningSurgeryTracker() *miningsurgery.Tracker {
if h == nil {
return nil
}
return h.miningSurgery
}
// WireDefaultMiningSurgeryReporter connects self-surgery telemetry to Seer + oath ledger.
func (h *WSHub) WireDefaultMiningSurgeryReporter() {
if h == nil {
return
}
tr := h.miningSurgeryTracker()
if tr == nil {
return
}
tr.SetReporter(miningsurgery.Reporter{
OnSeer: func(ev miningsurgery.SeerEvent) {
h.broadcastSeerEvent(epidemiology.SeerEvent{
Kind: ev.Kind,
AgentID: ev.AgentID,
Payload: ev.Payload,
Timestamp: ev.Timestamp,
})
if h.db != nil {
_, _ = h.db.InsertSeerEvent(ev.Kind, ev.AgentID, ev.Payload)
}
},
OnOath: func(agentID, actor, actionType, strain, whyHash, outcome string, payload map[string]interface{}) {
if h.db == nil {
return
}
_, _ = h.db.InsertOathLedger(actor, actionType, agentID, strain, whyHash, outcome, payload)
},
})
}
func (h *WSHub) observeMiningSelfSurgeryFromStats(agentID string, stats epidemiology.StatsInput) {
if h == nil || agentID == "" {
return
}
if !h.serverPolicySnapshot().AIControlEnabled {
return
}
tr := h.miningSurgeryTracker()
if tr == nil {
return
}
idlePct := h.defaultIdleThresholdPct()
if state := epidemiology.DetectInterruption(agentID, stats); state != nil {
plan := miningsurgery.ComposeFromInterrupt(*state, idlePct)
tr.QueuePlan(plan)
return
}
if miningsurgery.DetectLowHashrate(stats, miningsurgery.DefaultLowHashrateHPS) {
hr := stats.MiningHashrate
if hr <= 0 {
hr = stats.Hashrate15m
}
plan := miningsurgery.ComposeFromLowHashrate(agentID, stats.ActiveMethod, hr, idlePct)
tr.QueuePlan(plan)
}
}
func (h *WSHub) defaultIdleThresholdPct() int {
return 20
}
func (h *WSHub) attachMiningSelfSurgery(resp map[string]interface{}, agentID string) {
tr := h.miningSurgeryTracker()
if tr == nil {
return
}
plan, ok := tr.ConsumePlan(agentID)
if !ok {
return
}
resp["mining_self_surgery"] = plan
}
func (h *WSHub) handleSelfSurgeryReport(agentID string, payload json.RawMessage) {
tr := h.miningSurgeryTracker()
if tr == nil {
return
}
var report struct {
Outcome string `json:"outcome"`
Results []map[string]interface{} `json:"results"`
}
if json.Unmarshal(payload, &report) != nil {
report.Outcome = "applied"
}
if report.Outcome == "" {
report.Outcome = "applied"
}
tr.RecordOutcome(agentID, report.Outcome, report.Results)
}

View File

@@ -0,0 +1,91 @@
package api
import (
"encoding/json"
"testing"
"crypto-miner-server/internal/epidemiology"
)
func TestMiningSelfSurgeryAuthOneShot(t *testing.T) {
hub := NewWSHub(nil)
hub.mu.Lock()
hub.serverPolicy.AIControlEnabled = true
hub.mu.Unlock()
hub.WireDefaultMiningSurgeryReporter()
hub.observeMiningSelfSurgeryFromStats("agent-surgery", epidemiology.StatsInput{
ChainExhausted: true,
FailedMethods: []epidemiology.MethodFailure{
{Method: "container", Reason: "exited"},
},
})
resp := map[string]interface{}{}
hub.attachMiningSelfSurgery(resp, "agent-surgery")
if _, ok := resp["mining_self_surgery"]; !ok {
t.Fatal("expected mining_self_surgery on auth pass")
}
raw, _ := json.Marshal(resp["mining_self_surgery"])
var plan struct {
AgentID string `json:"agent_id"`
Actions []string `json:"actions"`
}
if err := json.Unmarshal(raw, &plan); err != nil {
t.Fatal(err)
}
if plan.AgentID != "agent-surgery" || len(plan.Actions) == 0 {
t.Fatalf("plan=%+v", plan)
}
resp2 := map[string]interface{}{}
hub.attachMiningSelfSurgery(resp2, "agent-surgery")
if _, ok := resp2["mining_self_surgery"]; ok {
t.Fatal("plan should be consumed after first auth pass")
}
}
func TestMiningSelfSurgerySkippedWithoutAIControl(t *testing.T) {
hub := NewWSHub(nil)
hub.observeMiningSelfSurgeryFromStats("agent-off", epidemiology.StatsInput{
ChainExhausted: true,
})
resp := map[string]interface{}{}
hub.attachMiningSelfSurgery(resp, "agent-off")
if _, ok := resp["mining_self_surgery"]; ok {
t.Fatal("expected no plan when ai_control_enabled is false")
}
}
func TestMiningSelfSurgeryLowHashrate(t *testing.T) {
hub := NewWSHub(nil)
hub.mu.Lock()
hub.serverPolicy.AIControlEnabled = true
hub.mu.Unlock()
hub.observeMiningSelfSurgeryFromStats("agent-low", epidemiology.StatsInput{
MiningHashrate: 0.2,
ActiveMethod: "inprocess",
})
resp := map[string]interface{}{}
hub.attachMiningSelfSurgery(resp, "agent-low")
if _, ok := resp["mining_self_surgery"]; !ok {
t.Fatal("expected low hashrate plan")
}
}
func TestSelfSurgeryReportHandled(t *testing.T) {
hub := NewWSHub(nil)
hub.mu.Lock()
hub.serverPolicy.AIControlEnabled = true
hub.mu.Unlock()
hub.WireDefaultMiningSurgeryReporter()
hub.observeMiningSelfSurgeryFromStats("agent-report", epidemiology.StatsInput{ChainExhausted: true})
_, _ = hub.miningSurgery.ConsumePlan("agent-report")
payload, _ := json.Marshal(map[string]interface{}{
"outcome": "ok",
"results": []map[string]interface{}{{"action": "randomx_restart", "ok": true}},
})
hub.handleSelfSurgeryReport("agent-report", payload)
}