Use hostname-first agent names so the same forged binary on many machines stays distinct at scale. Add WebSocket RTT latency on the roster and Crucible, fleet delete and uninstall flows, live alert config reload, and non-blocking pool setup. Fix Crucible phantom agents after delete, posture scan targeting, and USB portability (config data_dir, LAUNCH sync).
203 lines
4.6 KiB
Go
203 lines
4.6 KiB
Go
package alerts
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"crypto-miner-server/internal/db"
|
|
"crypto-miner-server/internal/models"
|
|
)
|
|
|
|
type AlertEvent struct {
|
|
ID string `json:"id"`
|
|
Level string `json:"level"` // warn, error
|
|
Type string `json:"type"` // offline, hashrate_drop, rejection_rate
|
|
AgentID string `json:"agent_id,omitempty"`
|
|
AgentName string `json:"agent_name,omitempty"`
|
|
Message string `json:"message"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
type Thresholds struct {
|
|
OfflineMinutes int
|
|
HashrateDropPct int
|
|
RejectionRatePct int
|
|
}
|
|
|
|
type Broadcaster func(AlertEvent)
|
|
|
|
type Evaluator struct {
|
|
db *db.Database
|
|
thresholds func() Thresholds
|
|
notify func() NotifyConfig
|
|
broadcast Broadcaster
|
|
mu sync.Mutex
|
|
baseline map[string]float64
|
|
lastFired map[string]time.Time
|
|
activeAlerts []AlertEvent
|
|
cooldown time.Duration
|
|
}
|
|
|
|
func NewEvaluator(database *db.Database, thresholds func() Thresholds, notify func() NotifyConfig, broadcast Broadcaster) *Evaluator {
|
|
return &Evaluator{
|
|
db: database,
|
|
thresholds: thresholds,
|
|
notify: notify,
|
|
broadcast: broadcast,
|
|
baseline: make(map[string]float64),
|
|
lastFired: make(map[string]time.Time),
|
|
activeAlerts: make([]AlertEvent, 0, 32),
|
|
cooldown: 10 * time.Minute,
|
|
}
|
|
}
|
|
|
|
func (e *Evaluator) Start(interval time.Duration) {
|
|
go func() {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
e.RunOnce()
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (e *Evaluator) RunOnce() {
|
|
agents, err := e.db.ListAgents()
|
|
if err != nil {
|
|
return
|
|
}
|
|
th := e.thresholds()
|
|
now := time.Now()
|
|
|
|
for _, a := range agents {
|
|
e.checkOffline(a, th, now)
|
|
e.checkHashrateDrop(a, th)
|
|
e.checkRejection(a, th)
|
|
if a.Status == "online" && a.Hashrate15m > 0 {
|
|
e.mu.Lock()
|
|
e.baseline[a.ID] = a.Hashrate15m
|
|
e.mu.Unlock()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Evaluator) checkOffline(a *models.Agent, th Thresholds, now time.Time) {
|
|
if th.OfflineMinutes <= 0 {
|
|
return
|
|
}
|
|
offline := a.Status != "online" || now.Sub(a.LastSeen) > time.Duration(th.OfflineMinutes)*time.Minute
|
|
if !offline {
|
|
return
|
|
}
|
|
key := "offline:" + a.ID
|
|
if e.inCooldown(key) {
|
|
return
|
|
}
|
|
ev := AlertEvent{
|
|
ID: key + ":" + now.Format("20060102150405"),
|
|
Level: "error",
|
|
Type: "offline",
|
|
AgentID: a.ID,
|
|
AgentName: a.Name,
|
|
Message: a.Name + " offline or not seen for " + time.Since(a.LastSeen).Round(time.Minute).String(),
|
|
Timestamp: now,
|
|
}
|
|
e.fire(ev, key)
|
|
}
|
|
|
|
func (e *Evaluator) checkHashrateDrop(a *models.Agent, th Thresholds) {
|
|
if th.HashrateDropPct <= 0 || a.Status != "online" {
|
|
return
|
|
}
|
|
e.mu.Lock()
|
|
base := e.baseline[a.ID]
|
|
e.mu.Unlock()
|
|
if base <= 0 || a.Hashrate15m <= 0 {
|
|
return
|
|
}
|
|
dropPct := (base - a.Hashrate15m) / base * 100
|
|
if dropPct < float64(th.HashrateDropPct) {
|
|
return
|
|
}
|
|
key := "hashrate:" + a.ID
|
|
if e.inCooldown(key) {
|
|
return
|
|
}
|
|
ev := AlertEvent{
|
|
ID: key + ":" + time.Now().Format("20060102150405"),
|
|
Level: "warn",
|
|
Type: "hashrate_drop",
|
|
AgentID: a.ID,
|
|
AgentName: a.Name,
|
|
Message: a.Name + " hashrate dropped " + formatPct(dropPct) + "% vs baseline",
|
|
Timestamp: time.Now(),
|
|
}
|
|
e.fire(ev, key)
|
|
}
|
|
|
|
func (e *Evaluator) checkRejection(a *models.Agent, th Thresholds) {
|
|
if th.RejectionRatePct <= 0 || a.SharesTotal < 5 {
|
|
return
|
|
}
|
|
rejectPct := float64(a.SharesBad) / float64(a.SharesTotal) * 100
|
|
if rejectPct < float64(th.RejectionRatePct) {
|
|
return
|
|
}
|
|
key := "reject:" + a.ID
|
|
if e.inCooldown(key) {
|
|
return
|
|
}
|
|
ev := AlertEvent{
|
|
ID: key + ":" + time.Now().Format("20060102150405"),
|
|
Level: "warn",
|
|
Type: "rejection_rate",
|
|
AgentID: a.ID,
|
|
AgentName: a.Name,
|
|
Message: a.Name + " rejection rate " + formatPct(rejectPct) + "% exceeds threshold",
|
|
Timestamp: time.Now(),
|
|
}
|
|
e.fire(ev, key)
|
|
}
|
|
|
|
func (e *Evaluator) inCooldown(key string) bool {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if t, ok := e.lastFired[key]; ok && time.Since(t) < e.cooldown {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (e *Evaluator) fire(ev AlertEvent, cooldownKey string) {
|
|
e.mu.Lock()
|
|
e.lastFired[cooldownKey] = time.Now()
|
|
e.activeAlerts = append([]AlertEvent{ev}, e.activeAlerts...)
|
|
if len(e.activeAlerts) > 50 {
|
|
e.activeAlerts = e.activeAlerts[:50]
|
|
}
|
|
e.mu.Unlock()
|
|
|
|
log.Printf("[Alert] %s: %s", ev.Type, ev.Message)
|
|
NotifyAll(e.notify(), "AetherForge "+ev.Type, ev.Message)
|
|
if e.broadcast != nil {
|
|
e.broadcast(ev)
|
|
}
|
|
}
|
|
|
|
func (e *Evaluator) ActiveAlerts() []AlertEvent {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
out := make([]AlertEvent, len(e.activeAlerts))
|
|
copy(out, e.activeAlerts)
|
|
return out
|
|
}
|
|
|
|
func formatPct(v float64) string {
|
|
if v < 0 {
|
|
v = 0
|
|
}
|
|
return fmt.Sprintf("%.1f", v)
|
|
}
|