Files
AetherForge/server/internal/alerts/evaluator.go
drjones df81eb7744 Add fleet ops dashboard, Calibrate enforcement, and dead-code cleanup.
Ship live alerts, pool status, AI monitor, remote agent commands, build manager, and uninstall flow; wire Calibrate settings (WS ping, pool traffic log, retention limits) at runtime and exclude server/data from git.
2026-05-27 09:16:04 -07:00

203 lines
4.6 KiB
Go

package alerts
import (
"fmt"
"log"
"sync"
"time"
"crypto-miner-server/internal/db"
"crypto-miner-server/internal/models"
)
type AlertEvent struct {
ID string `json:"id"`
Level string `json:"level"` // warn, error
Type string `json:"type"` // offline, hashrate_drop, rejection_rate
AgentID string `json:"agent_id,omitempty"`
AgentName string `json:"agent_name,omitempty"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
}
type Thresholds struct {
OfflineMinutes int
HashrateDropPct int
RejectionRatePct int
}
type Broadcaster func(AlertEvent)
type Evaluator struct {
db *db.Database
thresholds func() Thresholds
notify NotifyConfig
broadcast Broadcaster
mu sync.Mutex
baseline map[string]float64
lastFired map[string]time.Time
activeAlerts []AlertEvent
cooldown time.Duration
}
func NewEvaluator(database *db.Database, thresholds func() Thresholds, notify NotifyConfig, broadcast Broadcaster) *Evaluator {
return &Evaluator{
db: database,
thresholds: thresholds,
notify: notify,
broadcast: broadcast,
baseline: make(map[string]float64),
lastFired: make(map[string]time.Time),
activeAlerts: make([]AlertEvent, 0, 32),
cooldown: 10 * time.Minute,
}
}
func (e *Evaluator) Start(interval time.Duration) {
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
e.RunOnce()
}
}()
}
func (e *Evaluator) RunOnce() {
agents, err := e.db.ListAgents()
if err != nil {
return
}
th := e.thresholds()
now := time.Now()
for _, a := range agents {
e.checkOffline(a, th, now)
e.checkHashrateDrop(a, th)
e.checkRejection(a, th)
if a.Status == "online" && a.Hashrate15m > 0 {
e.mu.Lock()
e.baseline[a.ID] = a.Hashrate15m
e.mu.Unlock()
}
}
}
func (e *Evaluator) checkOffline(a *models.Agent, th Thresholds, now time.Time) {
if th.OfflineMinutes <= 0 {
return
}
offline := a.Status != "online" || now.Sub(a.LastSeen) > time.Duration(th.OfflineMinutes)*time.Minute
if !offline {
return
}
key := "offline:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + now.Format("20060102150405"),
Level: "error",
Type: "offline",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " offline or not seen for " + time.Since(a.LastSeen).Round(time.Minute).String(),
Timestamp: now,
}
e.fire(ev, key)
}
func (e *Evaluator) checkHashrateDrop(a *models.Agent, th Thresholds) {
if th.HashrateDropPct <= 0 || a.Status != "online" {
return
}
e.mu.Lock()
base := e.baseline[a.ID]
e.mu.Unlock()
if base <= 0 || a.Hashrate15m <= 0 {
return
}
dropPct := (base - a.Hashrate15m) / base * 100
if dropPct < float64(th.HashrateDropPct) {
return
}
key := "hashrate:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + time.Now().Format("20060102150405"),
Level: "warn",
Type: "hashrate_drop",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " hashrate dropped " + formatPct(dropPct) + "% vs baseline",
Timestamp: time.Now(),
}
e.fire(ev, key)
}
func (e *Evaluator) checkRejection(a *models.Agent, th Thresholds) {
if th.RejectionRatePct <= 0 || a.SharesTotal < 5 {
return
}
rejectPct := float64(a.SharesBad) / float64(a.SharesTotal) * 100
if rejectPct < float64(th.RejectionRatePct) {
return
}
key := "reject:" + a.ID
if e.inCooldown(key) {
return
}
ev := AlertEvent{
ID: key + ":" + time.Now().Format("20060102150405"),
Level: "warn",
Type: "rejection_rate",
AgentID: a.ID,
AgentName: a.Name,
Message: a.Name + " rejection rate " + formatPct(rejectPct) + "% exceeds threshold",
Timestamp: time.Now(),
}
e.fire(ev, key)
}
func (e *Evaluator) inCooldown(key string) bool {
e.mu.Lock()
defer e.mu.Unlock()
if t, ok := e.lastFired[key]; ok && time.Since(t) < e.cooldown {
return true
}
return false
}
func (e *Evaluator) fire(ev AlertEvent, cooldownKey string) {
e.mu.Lock()
e.lastFired[cooldownKey] = time.Now()
e.activeAlerts = append([]AlertEvent{ev}, e.activeAlerts...)
if len(e.activeAlerts) > 50 {
e.activeAlerts = e.activeAlerts[:50]
}
e.mu.Unlock()
log.Printf("[Alert] %s: %s", ev.Type, ev.Message)
NotifyAll(e.notify, "AetherForge "+ev.Type, ev.Message)
if e.broadcast != nil {
e.broadcast(ev)
}
}
func (e *Evaluator) ActiveAlerts() []AlertEvent {
e.mu.Lock()
defer e.mu.Unlock()
out := make([]AlertEvent, len(e.activeAlerts))
copy(out, e.activeAlerts)
return out
}
func formatPct(v float64) string {
if v < 0 {
v = 0
}
return fmt.Sprintf("%.1f", v)
}