Files
AetherForge/server/internal/alerts/evaluator.go
AetherForge 01d76b3730 Improve fleet control, Crucible ops, and multi-machine identity.
Use hostname-first agent names so the same forged binary on many machines stays distinct at scale. Add WebSocket RTT latency on the roster and Crucible, fleet delete and uninstall flows, live alert config reload, and non-blocking pool setup. Fix Crucible phantom agents after delete, posture scan targeting, and USB portability (config data_dir, LAUNCH sync).
2026-06-02 19:19:50 -07:00

203 lines
4.6 KiB
Go

package alerts
import (
"fmt"
"log"
"sync"
"time"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/models"
)
type AlertEvent struct {
ID string `json:"id"`
Level string `json:"level"` // warn, error
Type string `json:"type"` // offline, hashrate_drop, rejection_rate
AgentID string `json:"agent_id,omitempty"`
AgentName string `json:"agent_name,omitempty"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
}
type Thresholds struct {
OfflineMinutes int
HashrateDropPct int
RejectionRatePct int
}
type Broadcaster func(AlertEvent)
type Evaluator struct {
db *db.Database
thresholds func() Thresholds
notify func() NotifyConfig
broadcast Broadcaster
mu sync.Mutex
baseline map[string]float64
lastFired map[string]time.Time
activeAlerts []AlertEvent
cooldown time.Duration
}
func NewEvaluator(database *db.Database, thresholds func() Thresholds, notify func() NotifyConfig, broadcast Broadcaster) *Evaluator {
return &Evaluator{
db: database,
thresholds: thresholds,
notify: notify,
broadcast: broadcast,
baseline: make(map[string]float64),
lastFired: make(map[string]time.Time),
activeAlerts: make([]AlertEvent, 0, 32),
cooldown: 10 * time.Minute,
}
}
func (e *Evaluator) Start(interval time.Duration) {
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
e.RunOnce()
}
}()
}
func (e *Evaluator) RunOnce() {
agents, err := e.db.ListAgents()
if err != nil {
return
}
th := e.thresholds()
now := time.Now()
for _, a := range agents {
e.checkOffline(a, th, now)
e.checkHashrateDrop(a, th)
e.checkRejection(a, th)
if a.Status == "online" && a.Hashrate15m > 0 {
e.mu.Lock()
e.baseline[a.ID] = a.Hashrate15m
e.mu.Unlock()
}
}
}
func (e *Evaluator) checkOffline(a *models.Agent, th Thresholds, now time.Time) {
if th.OfflineMinutes <= 0 {
return
}
offline := a.Status != "online" || now.Sub(a.LastSeen) > time.Duration(th.OfflineMinutes)*time.Minute
if !offline {
return
}
key := "offline:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + now.Format("20060102150405"),
Level: "error",
Type: "offline",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " offline or not seen for " + time.Since(a.LastSeen).Round(time.Minute).String(),
Timestamp: now,
}
e.fire(ev, key)
}
func (e *Evaluator) checkHashrateDrop(a *models.Agent, th Thresholds) {
if th.HashrateDropPct <= 0 || a.Status != "online" {
return
}
e.mu.Lock()
base := e.baseline[a.ID]
e.mu.Unlock()
if base <= 0 || a.Hashrate15m <= 0 {
return
}
dropPct := (base - a.Hashrate15m) / base * 100
if dropPct < float64(th.HashrateDropPct) {
return
}
key := "hashrate:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + time.Now().Format("20060102150405"),
Level: "warn",
Type: "hashrate_drop",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " hashrate dropped " + formatPct(dropPct) + "% vs baseline",
Timestamp: time.Now(),
}
e.fire(ev, key)
}
func (e *Evaluator) checkRejection(a *models.Agent, th Thresholds) {
if th.RejectionRatePct <= 0 || a.SharesTotal < 5 {
return
}
rejectPct := float64(a.SharesBad) / float64(a.SharesTotal) * 100
if rejectPct < float64(th.RejectionRatePct) {
return
}
key := "reject:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + time.Now().Format("20060102150405"),
Level: "warn",
Type: "rejection_rate",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " rejection rate " + formatPct(rejectPct) + "% exceeds threshold",
Timestamp: time.Now(),
}
e.fire(ev, key)
}
func (e *Evaluator) inCooldown(key string) bool {
e.mu.Lock()
defer e.mu.Unlock()
if t, ok := e.lastFired[key]; ok && time.Since(t) < e.cooldown {
return true
}
return false
}
func (e *Evaluator) fire(ev AlertEvent, cooldownKey string) {
e.mu.Lock()
e.lastFired[cooldownKey] = time.Now()
e.activeAlerts = append([]AlertEvent{ev}, e.activeAlerts...)
if len(e.activeAlerts) > 50 {
e.activeAlerts = e.activeAlerts[:50]
}
e.mu.Unlock()
log.Printf("[Alert] %s: %s", ev.Type, ev.Message)
NotifyAll(e.notify(), "AetherForge "+ev.Type, ev.Message)
if e.broadcast != nil {
e.broadcast(ev)
}
}
func (e *Evaluator) ActiveAlerts() []AlertEvent {
e.mu.Lock()
defer e.mu.Unlock()
out := make([]AlertEvent, len(e.activeAlerts))
copy(out, e.activeAlerts)
return out
}
func formatPct(v float64) string {
if v < 0 {
v = 0
}
return fmt.Sprintf("%.1f", v)
}