Add fleet ops dashboard, Calibrate enforcement, and dead-code cleanup.
Ship live alerts, pool status, AI monitor, remote agent commands, build manager, and uninstall flow; wire Calibrate settings (WS ping, pool traffic log, retention limits) at runtime and exclude server/data from git.
This commit is contained in:
drjones
202
server/internal/alerts/evaluator.go
Normal file
202
server/internal/alerts/evaluator.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"crypto-miner-server/internal/db"
|
||||
"crypto-miner-server/internal/models"
|
||||
)
|
||||
|
||||
type AlertEvent struct {
|
||||
ID string `json:"id"`
|
||||
Level string `json:"level"` // warn, error
|
||||
Type string `json:"type"` // offline, hashrate_drop, rejection_rate
|
||||
AgentID string `json:"agent_id,omitempty"`
|
||||
AgentName string `json:"agent_name,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
type Thresholds struct {
|
||||
OfflineMinutes int
|
||||
HashrateDropPct int
|
||||
RejectionRatePct int
|
||||
}
|
||||
|
||||
type Broadcaster func(AlertEvent)
|
||||
|
||||
type Evaluator struct {
|
||||
db *db.Database
|
||||
thresholds func() Thresholds
|
||||
notify NotifyConfig
|
||||
broadcast Broadcaster
|
||||
mu sync.Mutex
|
||||
baseline map[string]float64
|
||||
lastFired map[string]time.Time
|
||||
activeAlerts []AlertEvent
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewEvaluator(database *db.Database, thresholds func() Thresholds, notify NotifyConfig, broadcast Broadcaster) *Evaluator {
|
||||
return &Evaluator{
|
||||
db: database,
|
||||
thresholds: thresholds,
|
||||
notify: notify,
|
||||
broadcast: broadcast,
|
||||
baseline: make(map[string]float64),
|
||||
lastFired: make(map[string]time.Time),
|
||||
activeAlerts: make([]AlertEvent, 0, 32),
|
||||
cooldown: 10 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Evaluator) Start(interval time.Duration) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
e.RunOnce()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (e *Evaluator) RunOnce() {
|
||||
agents, err := e.db.ListAgents()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
th := e.thresholds()
|
||||
now := time.Now()
|
||||
|
||||
for _, a := range agents {
|
||||
e.checkOffline(a, th, now)
|
||||
e.checkHashrateDrop(a, th)
|
||||
e.checkRejection(a, th)
|
||||
if a.Status == "online" && a.Hashrate15m > 0 {
|
||||
e.mu.Lock()
|
||||
e.baseline[a.ID] = a.Hashrate15m
|
||||
e.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Evaluator) checkOffline(a *models.Agent, th Thresholds, now time.Time) {
|
||||
if th.OfflineMinutes <= 0 {
|
||||
return
|
||||
}
|
||||
offline := a.Status != "online" || now.Sub(a.LastSeen) > time.Duration(th.OfflineMinutes)*time.Minute
|
||||
if !offline {
|
||||
return
|
||||
}
|
||||
key := "offline:" + a.ID
|
||||
if e.inCooldown(key) {
|
||||
return
|
||||
}
|
||||
ev := AlertEvent{
|
||||
ID: key + ":" + now.Format("20060102150405"),
|
||||
Level: "error",
|
||||
Type: "offline",
|
||||
AgentID: a.ID,
|
||||
AgentName: a.Name,
|
||||
Message: a.Name + " offline or not seen for " + time.Since(a.LastSeen).Round(time.Minute).String(),
|
||||
Timestamp: now,
|
||||
}
|
||||
e.fire(ev, key)
|
||||
}
|
||||
|
||||
func (e *Evaluator) checkHashrateDrop(a *models.Agent, th Thresholds) {
|
||||
if th.HashrateDropPct <= 0 || a.Status != "online" {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
base := e.baseline[a.ID]
|
||||
e.mu.Unlock()
|
||||
if base <= 0 || a.Hashrate15m <= 0 {
|
||||
return
|
||||
}
|
||||
dropPct := (base - a.Hashrate15m) / base * 100
|
||||
if dropPct < float64(th.HashrateDropPct) {
|
||||
return
|
||||
}
|
||||
key := "hashrate:" + a.ID
|
||||
if e.inCooldown(key) {
|
||||
return
|
||||
}
|
||||
ev := AlertEvent{
|
||||
ID: key + ":" + time.Now().Format("20060102150405"),
|
||||
Level: "warn",
|
||||
Type: "hashrate_drop",
|
||||
AgentID: a.ID,
|
||||
AgentName: a.Name,
|
||||
Message: a.Name + " hashrate dropped " + formatPct(dropPct) + "% vs baseline",
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
e.fire(ev, key)
|
||||
}
|
||||
|
||||
func (e *Evaluator) checkRejection(a *models.Agent, th Thresholds) {
|
||||
if th.RejectionRatePct <= 0 || a.SharesTotal < 5 {
|
||||
return
|
||||
}
|
||||
rejectPct := float64(a.SharesBad) / float64(a.SharesTotal) * 100
|
||||
if rejectPct < float64(th.RejectionRatePct) {
|
||||
return
|
||||
}
|
||||
key := "reject:" + a.ID
|
||||
if e.inCooldown(key) {
|
||||
return
|
||||
}
|
||||
ev := AlertEvent{
|
||||
ID: key + ":" + time.Now().Format("20060102150405"),
|
||||
Level: "warn",
|
||||
Type: "rejection_rate",
|
||||
AgentID: a.ID,
|
||||
AgentName: a.Name,
|
||||
Message: a.Name + " rejection rate " + formatPct(rejectPct) + "% exceeds threshold",
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
e.fire(ev, key)
|
||||
}
|
||||
|
||||
func (e *Evaluator) inCooldown(key string) bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if t, ok := e.lastFired[key]; ok && time.Since(t) < e.cooldown {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *Evaluator) fire(ev AlertEvent, cooldownKey string) {
|
||||
e.mu.Lock()
|
||||
e.lastFired[cooldownKey] = time.Now()
|
||||
e.activeAlerts = append([]AlertEvent{ev}, e.activeAlerts...)
|
||||
if len(e.activeAlerts) > 50 {
|
||||
e.activeAlerts = e.activeAlerts[:50]
|
||||
}
|
||||
e.mu.Unlock()
|
||||
|
||||
log.Printf("[Alert] %s: %s", ev.Type, ev.Message)
|
||||
NotifyAll(e.notify, "AetherForge "+ev.Type, ev.Message)
|
||||
if e.broadcast != nil {
|
||||
e.broadcast(ev)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Evaluator) ActiveAlerts() []AlertEvent {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
out := make([]AlertEvent, len(e.activeAlerts))
|
||||
copy(out, e.activeAlerts)
|
||||
return out
|
||||
}
|
||||
|
||||
func formatPct(v float64) string {
|
||||
if v < 0 {
|
||||
v = 0
|
||||
}
|
||||
return fmt.Sprintf("%.1f", v)
|
||||
}
|
||||
40
server/internal/alerts/evaluator_test.go
Normal file
40
server/internal/alerts/evaluator_test.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"crypto-miner-server/internal/models"
|
||||
)
|
||||
|
||||
func TestFormatPct(t *testing.T) {
|
||||
if formatPct(50.55) != "50.5" {
|
||||
t.Fatalf("expected 50.5 got %s", formatPct(50.55))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluatorOfflineAlert(t *testing.T) {
|
||||
var fired []AlertEvent
|
||||
e := &Evaluator{
|
||||
thresholds: func() Thresholds { return Thresholds{OfflineMinutes: 5} },
|
||||
broadcast: func(ev AlertEvent) { fired = append(fired, ev) },
|
||||
baseline: make(map[string]float64),
|
||||
lastFired: make(map[string]time.Time),
|
||||
cooldown: 0,
|
||||
}
|
||||
|
||||
agent := &models.Agent{
|
||||
ID: "a1",
|
||||
Name: "worker-1",
|
||||
Status: "offline",
|
||||
LastSeen: time.Now().Add(-10 * time.Minute),
|
||||
}
|
||||
|
||||
e.checkOffline(agent, e.thresholds(), time.Now())
|
||||
if len(fired) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(fired))
|
||||
}
|
||||
if fired[0].Type != "offline" {
|
||||
t.Fatalf("expected offline alert")
|
||||
}
|
||||
}
|
||||
83
server/internal/alerts/notify.go
Normal file
83
server/internal/alerts/notify.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type NotifyConfig struct {
|
||||
TelegramBotToken string
|
||||
TelegramChatID string
|
||||
EmailEnabled bool
|
||||
SMTPHost string
|
||||
SMTPPort int
|
||||
SMTPUser string
|
||||
SMTPPassword string
|
||||
EmailTo string
|
||||
EmailFrom string
|
||||
}
|
||||
|
||||
func SendTelegram(cfg NotifyConfig, text string) error {
|
||||
if cfg.TelegramBotToken == "" || cfg.TelegramChatID == "" {
|
||||
return nil
|
||||
}
|
||||
url := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", cfg.TelegramBotToken)
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"chat_id": cfg.TelegramChatID,
|
||||
"text": text,
|
||||
})
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("telegram API status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SendEmail(cfg NotifyConfig, subject, body string) error {
|
||||
if !cfg.EmailEnabled || cfg.SMTPHost == "" || cfg.EmailTo == "" {
|
||||
return nil
|
||||
}
|
||||
from := cfg.EmailFrom
|
||||
if from == "" {
|
||||
from = cfg.SMTPUser
|
||||
}
|
||||
port := cfg.SMTPPort
|
||||
if port <= 0 {
|
||||
port = 587
|
||||
}
|
||||
addr := fmt.Sprintf("%s:%d", cfg.SMTPHost, port)
|
||||
msg := strings.Join([]string{
|
||||
fmt.Sprintf("From: %s", from),
|
||||
fmt.Sprintf("To: %s", cfg.EmailTo),
|
||||
fmt.Sprintf("Subject: %s", subject),
|
||||
"MIME-Version: 1.0",
|
||||
"Content-Type: text/plain; charset=UTF-8",
|
||||
"",
|
||||
body,
|
||||
}, "\r\n")
|
||||
var auth smtp.Auth
|
||||
if cfg.SMTPUser != "" {
|
||||
auth = smtp.PlainAuth("", cfg.SMTPUser, cfg.SMTPPassword, cfg.SMTPHost)
|
||||
}
|
||||
return smtp.SendMail(addr, auth, from, []string{cfg.EmailTo}, []byte(msg))
|
||||
}
|
||||
|
||||
func NotifyAll(cfg NotifyConfig, subject, text string) {
|
||||
_ = SendTelegram(cfg, subject+": "+text)
|
||||
_ = SendEmail(cfg, subject, text)
|
||||
}
|
||||
Reference in New Issue
Block a user