Add mining self-surgery for on-host recovery when AI control detects stalls.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
This commit is contained in:
107
server/internal/api/mining_self_surgery_bridge.go
Normal file
107
server/internal/api/mining_self_surgery_bridge.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"crypto-miner-server/internal/epidemiology"
|
||||
"crypto-miner-server/internal/miningsurgery"
|
||||
)
|
||||
|
||||
func (h *WSHub) miningSurgeryTracker() *miningsurgery.Tracker {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
return h.miningSurgery
|
||||
}
|
||||
|
||||
// WireDefaultMiningSurgeryReporter connects self-surgery telemetry to Seer + oath ledger.
|
||||
func (h *WSHub) WireDefaultMiningSurgeryReporter() {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
tr := h.miningSurgeryTracker()
|
||||
if tr == nil {
|
||||
return
|
||||
}
|
||||
tr.SetReporter(miningsurgery.Reporter{
|
||||
OnSeer: func(ev miningsurgery.SeerEvent) {
|
||||
h.broadcastSeerEvent(epidemiology.SeerEvent{
|
||||
Kind: ev.Kind,
|
||||
AgentID: ev.AgentID,
|
||||
Payload: ev.Payload,
|
||||
Timestamp: ev.Timestamp,
|
||||
})
|
||||
if h.db != nil {
|
||||
_, _ = h.db.InsertSeerEvent(ev.Kind, ev.AgentID, ev.Payload)
|
||||
}
|
||||
},
|
||||
OnOath: func(agentID, actor, actionType, strain, whyHash, outcome string, payload map[string]interface{}) {
|
||||
if h.db == nil {
|
||||
return
|
||||
}
|
||||
_, _ = h.db.InsertOathLedger(actor, actionType, agentID, strain, whyHash, outcome, payload)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *WSHub) observeMiningSelfSurgeryFromStats(agentID string, stats epidemiology.StatsInput) {
|
||||
if h == nil || agentID == "" {
|
||||
return
|
||||
}
|
||||
if !h.serverPolicySnapshot().AIControlEnabled {
|
||||
return
|
||||
}
|
||||
tr := h.miningSurgeryTracker()
|
||||
if tr == nil {
|
||||
return
|
||||
}
|
||||
idlePct := h.defaultIdleThresholdPct()
|
||||
|
||||
if state := epidemiology.DetectInterruption(agentID, stats); state != nil {
|
||||
plan := miningsurgery.ComposeFromInterrupt(*state, idlePct)
|
||||
tr.QueuePlan(plan)
|
||||
return
|
||||
}
|
||||
if miningsurgery.DetectLowHashrate(stats, miningsurgery.DefaultLowHashrateHPS) {
|
||||
hr := stats.MiningHashrate
|
||||
if hr <= 0 {
|
||||
hr = stats.Hashrate15m
|
||||
}
|
||||
plan := miningsurgery.ComposeFromLowHashrate(agentID, stats.ActiveMethod, hr, idlePct)
|
||||
tr.QueuePlan(plan)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *WSHub) defaultIdleThresholdPct() int {
|
||||
return 20
|
||||
}
|
||||
|
||||
func (h *WSHub) attachMiningSelfSurgery(resp map[string]interface{}, agentID string) {
|
||||
tr := h.miningSurgeryTracker()
|
||||
if tr == nil {
|
||||
return
|
||||
}
|
||||
plan, ok := tr.ConsumePlan(agentID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
resp["mining_self_surgery"] = plan
|
||||
}
|
||||
|
||||
func (h *WSHub) handleSelfSurgeryReport(agentID string, payload json.RawMessage) {
|
||||
tr := h.miningSurgeryTracker()
|
||||
if tr == nil {
|
||||
return
|
||||
}
|
||||
var report struct {
|
||||
Outcome string `json:"outcome"`
|
||||
Results []map[string]interface{} `json:"results"`
|
||||
}
|
||||
if json.Unmarshal(payload, &report) != nil {
|
||||
report.Outcome = "applied"
|
||||
}
|
||||
if report.Outcome == "" {
|
||||
report.Outcome = "applied"
|
||||
}
|
||||
tr.RecordOutcome(agentID, report.Outcome, report.Results)
|
||||
}
|
||||
91
server/internal/api/mining_self_surgery_test.go
Normal file
91
server/internal/api/mining_self_surgery_test.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/epidemiology"
|
||||
)
|
||||
|
||||
func TestMiningSelfSurgeryAuthOneShot(t *testing.T) {
|
||||
hub := NewWSHub(nil)
|
||||
hub.mu.Lock()
|
||||
hub.serverPolicy.AIControlEnabled = true
|
||||
hub.mu.Unlock()
|
||||
hub.WireDefaultMiningSurgeryReporter()
|
||||
|
||||
hub.observeMiningSelfSurgeryFromStats("agent-surgery", epidemiology.StatsInput{
|
||||
ChainExhausted: true,
|
||||
FailedMethods: []epidemiology.MethodFailure{
|
||||
{Method: "container", Reason: "exited"},
|
||||
},
|
||||
})
|
||||
|
||||
resp := map[string]interface{}{}
|
||||
hub.attachMiningSelfSurgery(resp, "agent-surgery")
|
||||
if _, ok := resp["mining_self_surgery"]; !ok {
|
||||
t.Fatal("expected mining_self_surgery on auth pass")
|
||||
}
|
||||
raw, _ := json.Marshal(resp["mining_self_surgery"])
|
||||
var plan struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Actions []string `json:"actions"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.AgentID != "agent-surgery" || len(plan.Actions) == 0 {
|
||||
t.Fatalf("plan=%+v", plan)
|
||||
}
|
||||
|
||||
resp2 := map[string]interface{}{}
|
||||
hub.attachMiningSelfSurgery(resp2, "agent-surgery")
|
||||
if _, ok := resp2["mining_self_surgery"]; ok {
|
||||
t.Fatal("plan should be consumed after first auth pass")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiningSelfSurgerySkippedWithoutAIControl(t *testing.T) {
|
||||
hub := NewWSHub(nil)
|
||||
hub.observeMiningSelfSurgeryFromStats("agent-off", epidemiology.StatsInput{
|
||||
ChainExhausted: true,
|
||||
})
|
||||
resp := map[string]interface{}{}
|
||||
hub.attachMiningSelfSurgery(resp, "agent-off")
|
||||
if _, ok := resp["mining_self_surgery"]; ok {
|
||||
t.Fatal("expected no plan when ai_control_enabled is false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiningSelfSurgeryLowHashrate(t *testing.T) {
|
||||
hub := NewWSHub(nil)
|
||||
hub.mu.Lock()
|
||||
hub.serverPolicy.AIControlEnabled = true
|
||||
hub.mu.Unlock()
|
||||
|
||||
hub.observeMiningSelfSurgeryFromStats("agent-low", epidemiology.StatsInput{
|
||||
MiningHashrate: 0.2,
|
||||
ActiveMethod: "inprocess",
|
||||
})
|
||||
resp := map[string]interface{}{}
|
||||
hub.attachMiningSelfSurgery(resp, "agent-low")
|
||||
if _, ok := resp["mining_self_surgery"]; !ok {
|
||||
t.Fatal("expected low hashrate plan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelfSurgeryReportHandled(t *testing.T) {
|
||||
hub := NewWSHub(nil)
|
||||
hub.mu.Lock()
|
||||
hub.serverPolicy.AIControlEnabled = true
|
||||
hub.mu.Unlock()
|
||||
hub.WireDefaultMiningSurgeryReporter()
|
||||
hub.observeMiningSelfSurgeryFromStats("agent-report", epidemiology.StatsInput{ChainExhausted: true})
|
||||
_, _ = hub.miningSurgery.ConsumePlan("agent-report")
|
||||
|
||||
payload, _ := json.Marshal(map[string]interface{}{
|
||||
"outcome": "ok",
|
||||
"results": []map[string]interface{}{{"action": "randomx_restart", "ok": true}},
|
||||
})
|
||||
hub.handleSelfSurgeryReport("agent-report", payload)
|
||||
}
|
||||
Reference in New Issue
Block a user