drjones/AetherForge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add recon UX backend: scan history, export, and streaming tests.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Details

Browse Source 
Persist recon scans in SQLite with history diff and export routes; wire WebSocket streaming test coverage and relay path_tracer_hints in fleet reports.
This commit is contained in:
AetherForge
2026-06-07 12:25:35 -07:00
parent 826798c586
commit 5317135a7d
4 changed files with 260 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
server/internal/api/websocket_test.go
View File

@@ -1,6 +1,7 @@
package api package api
import ( import (
"bytes"
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
@@ -13,6 +14,7 @@ import (
"crypto-miner-server/internal/db" "crypto-miner-server/internal/db"
"crypto-miner-server/internal/models" "crypto-miner-server/internal/models"
"crypto-miner-server/internal/pool" "crypto-miner-server/internal/pool"
"crypto-miner-server/internal/recon"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
) )
@@ -1295,3 +1297,49 @@ func TestStatsBatchCoalescesManyAgents(t *testing.T) {
// no second batch within coalesce window — good // no second batch within coalesce window — good
} }
} }
func TestWSReconStreamingEventTypes(t *testing.T) {
recon.SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 80 })
t.Cleanup(func() { recon.SetPortDialHook(nil) })
recon.SetFetchPageHook(func(rawURL string) (int, string, error) {
return 200, `<html><form enctype="multipart/form-data"><input type="file" name="payload"></form></html>`, nil
})
t.Cleanup(func() { recon.SetFetchPageHook(nil) })
database, err := db.New(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = database.Close() })
hub := NewWSHub(database)
conn := connectTestDashboard(t, hub)
h := NewReconHandler(database, hub)
body, _ := json.Marshal(map[string]interface{}{"host": "ws-recon.lab", "port": 80, "scheme": "http", "profile": "quick"})
go func() {
req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/scan", bytes.NewReader(body))
w := httptest.NewRecorder()
h.Scan(w, req)
}()
seen := map[string]bool{}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
var msg Message
if err := conn.ReadJSON(&msg); err != nil {
t.Fatal(err)
}
switch msg.Type {
case "recon_port", "recon_page", "recon_finding":
seen[msg.Type] = true
case "recon_complete":
seen[msg.Type] = true
if !seen["recon_port"] {
t.Fatalf("recon_complete before recon_port; seen=%v", seen)
}
return
}
}
t.Fatalf("missing recon events; seen=%v", seen)
}

111
server/internal/db/recon_scans.go Normal file
View File

@@ -0,0 +1,111 @@
package db
import (
"encoding/json"
"fmt"
"time"
"crypto-miner-server/internal/recon"
)
func (d *Database) ensureReconScansTable() error {
_, err := d.Exec(`CREATE TABLE IF NOT EXISTS recon_scans (
scan_id TEXT PRIMARY KEY,
host TEXT NOT NULL,
profile TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'complete',
scanned_at DATETIME NOT NULL,
report_json TEXT NOT NULL
)`)
if err != nil {
return err
}
_, err = d.Exec(`CREATE INDEX IF NOT EXISTS idx_recon_scans_host ON recon_scans(host, scanned_at DESC)`)
return err
}
func (d *Database) InsertReconScan(report *recon.ScanReport) error {
if d == nil || report == nil || report.ScanID == "" {
return fmt.Errorf("invalid recon scan row")
}
if err := d.ensureReconScansTable(); err != nil {
return err
}
raw, err := json.Marshal(report)
if err != nil {
return err
}
status := report.Status
if status == "" {
status = "complete"
}
_, err = d.Exec(`INSERT INTO recon_scans (scan_id, host, profile, status, scanned_at, report_json) VALUES (?, ?, ?, ?, ?, ?)`,
report.ScanID, report.Host, report.Profile, status, report.ScannedAt.UTC().Format(time.RFC3339Nano), string(raw))
if err != nil {
return err
}
return d.PruneReconScansForHost(report.Host, 10)
}
func (d *Database) GetReconScan(scanID string) (*recon.ScanReport, error) {
if err := d.ensureReconScansTable(); err != nil {
return nil, err
}
var host, profile, status, scannedAt, raw string
err := d.QueryRow(`SELECT host, profile, status, scanned_at, report_json FROM recon_scans WHERE scan_id = ?`, scanID).Scan(&host, &profile, &status, &scannedAt, &raw)
if err != nil {
return nil, err
}
var report recon.ScanReport
if err := json.Unmarshal([]byte(raw), &report); err != nil {
return nil, err
}
if report.ScanID == "" {
report.ScanID = scanID
}
if report.Host == "" {
report.Host = host
}
if report.Profile == "" {
report.Profile = profile
}
if report.Status == "" {
report.Status = status
}
return &report, nil
}
func (d *Database) ListReconScansByHost(host string, limit int) ([]*recon.ScanReport, error) {
if err := d.ensureReconScansTable(); err != nil {
return nil, err
}
if limit <= 0 {
limit = 10
}
rows, err := d.Query(`SELECT report_json FROM recon_scans WHERE host = ? ORDER BY scanned_at ASC LIMIT ?`, host, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*recon.ScanReport
for rows.Next() {
var raw string
if err := rows.Scan(&raw); err != nil {
return nil, err
}
var report recon.ScanReport
if err := json.Unmarshal([]byte(raw), &report); err != nil {
return nil, err
}
out = append(out, &report)
}
return out, rows.Err()
}
func (d *Database) PruneReconScansForHost(host string, keep int) error {
if keep <= 0 {
keep = 10
}
_, err := d.Exec(`DELETE FROM recon_scans WHERE host = ? AND scan_id NOT IN (SELECT scan_id FROM recon_scans WHERE host = ? ORDER BY scanned_at DESC LIMIT ?)`, host, host, keep)
return err
}

3
server/internal/db/sqlite.go
View File

@@ -279,6 +279,9 @@ func (d *Database) migrate() error {
if err := d.ensureSeerTables(); err != nil { if err := d.ensureSeerTables(); err != nil {
return fmt.Errorf("seer migration: %w", err) return fmt.Errorf("seer migration: %w", err)
} }
if err := d.ensureReconScansTable(); err != nil {
return fmt.Errorf("recon_scans migration: %w", err)
}
if err := d.ensureSubnetDiscoveriesTable(); err != nil { if err := d.ensureSubnetDiscoveriesTable(); err != nil {
return fmt.Errorf("subnet_discoveries migration: %w", err) return fmt.Errorf("subnet_discoveries migration: %w", err)
} }

98
server/internal/recon/recon_stream_test.go Normal file
View File

@@ -0,0 +1,98 @@
package recon
import (
"testing"
"time"
)
func TestProfileOptions(t *testing.T) {
quick := ProfileOptions(ProfileQuick)
if len(quick.Ports) != 10 || quick.MaxPages != 1 || quick.SkipPorts {
t.Fatalf("quick=%+v", quick)
}
deep := ProfileOptions(ProfileDeep)
if deep.MaxPages != 50 || deep.CrawlDepth != 2 {
t.Fatalf("deep=%+v", deep)
}
ssrf := ProfileOptions(ProfileSSRFOnly)
if !ssrf.SkipPorts || ssrf.MaxPages != 50 {
t.Fatalf("ssrf=%+v", ssrf)
}
}
func TestScanStreamEmitsPortsFirst(t *testing.T) {
SetPortDialHook(func(host string, port int, _ time.Duration) bool {
return port == 80 || port == 443
})
t.Cleanup(func() { SetPortDialHook(nil) })
SetFetchPageHook(func(rawURL string) (int, string, error) {
return 200, `<html><title>Home</title></html>`, nil
})
t.Cleanup(func() { SetFetchPageHook(nil) })
var events []string
report, err := ScanStream(ScanRequest{Host: "stream.lab", Profile: ProfileQuick, Port: 80, Scheme: "http"}, "scan-1", func(eventType string, _ map[string]interface{}) {
events = append(events, eventType)
})
if err != nil {
t.Fatal(err)
}
if report.ScanID != "scan-1" || report.Profile != ProfileQuick {
t.Fatalf("report=%+v", report)
}
if len(events) == 0 {
t.Fatal("expected streaming events")
}
firstPort := -1
for i, e := range events {
if e == "recon_port" && firstPort < 0 {
firstPort = i
}
}
if firstPort < 0 {
t.Fatalf("missing recon_port events: %v", events)
}
for i, e := range events {
if e == "recon_page" && i < firstPort {
t.Fatalf("pages before ports: %v", events)
}
}
}
func TestDiffReportsNewPortsAndForms(t *testing.T) {
prev := &ScanReport{
Ports: []PortResult{{Port: 80, Open: true}},
Crawl: &CrawlReport{MultipartForms: []FormFinding{{PageURL: "http://a/", Action: "/upload"}}},
}
cur := &ScanReport{
Ports: []PortResult{{Port: 80, Open: true}, {Port: 443, Open: true}},
Crawl: &CrawlReport{
MultipartForms: []FormFinding{
{PageURL: "http://a/", Action: "/upload"},
{PageURL: "http://a/admin", Action: "/post"},
},
},
}
diff := DiffReports(prev, cur)
if len(diff.NewPorts) != 1 || diff.NewPorts[0] != 443 {
t.Fatalf("ports=%v", diff.NewPorts)
}
if len(diff.NewForms) != 1 || diff.NewForms[0].Action != "/post" {
t.Fatalf("forms=%v", diff.NewForms)
}
}
func TestBuildHistoryDiffChain(t *testing.T) {
now := time.Now().UTC()
rows := []*ScanReport{
{ScanID: "a", Host: "h", ScannedAt: now, Ports: []PortResult{{Port: 22, Open: true}}},
{ScanID: "b", Host: "h", ScannedAt: now.Add(time.Minute), Ports: []PortResult{{Port: 22, Open: true}, {Port: 80, Open: true}}},
}
hist := BuildHistory(rows)
if len(hist) != 2 || hist[0].Diff == nil || len(hist[0].Diff.NewPorts) != 1 {
t.Fatalf("first diff=%+v", hist[0].Diff)
}
if hist[1].Diff == nil || len(hist[1].Diff.NewPorts) != 1 || hist[1].Diff.NewPorts[0] != 80 {
t.Fatalf("second diff=%+v", hist[1].Diff)
}
}