From 5317135a7dca8f10c96c7cd645086af95e010f73 Mon Sep 17 00:00:00 2001 From: AetherForge Date: Sun, 7 Jun 2026 12:25:35 -0700 Subject: [PATCH] Add recon UX backend: scan history, export, and streaming tests. Persist recon scans in SQLite with history diff and export routes; wire WebSocket streaming test coverage and relay path_tracer_hints in fleet reports. --- server/internal/api/websocket_test.go | 48 +++++++++ server/internal/db/recon_scans.go | 111 +++++++++++++++++++++ server/internal/db/sqlite.go | 3 + server/internal/recon/recon_stream_test.go | 98 ++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 server/internal/db/recon_scans.go create mode 100644 server/internal/recon/recon_stream_test.go diff --git a/server/internal/api/websocket_test.go b/server/internal/api/websocket_test.go index 4151cd9..3f3c7c1 100644 --- a/server/internal/api/websocket_test.go +++ b/server/internal/api/websocket_test.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "encoding/base64" "encoding/json" "fmt" @@ -13,6 +14,7 @@ import ( "crypto-miner-server/internal/db" "crypto-miner-server/internal/models" "crypto-miner-server/internal/pool" + "crypto-miner-server/internal/recon" "github.com/gorilla/websocket" ) @@ -1295,3 +1297,49 @@ func TestStatsBatchCoalescesManyAgents(t *testing.T) { // no second batch within coalesce window — good } } + +func TestWSReconStreamingEventTypes(t *testing.T) { + recon.SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 80 }) + t.Cleanup(func() { recon.SetPortDialHook(nil) }) + recon.SetFetchPageHook(func(rawURL string) (int, string, error) { + return 200, `
`, nil + }) + t.Cleanup(func() { recon.SetFetchPageHook(nil) }) + + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + + hub := NewWSHub(database) + conn := connectTestDashboard(t, hub) + h := NewReconHandler(database, hub) + + body, _ := json.Marshal(map[string]interface{}{"host": "ws-recon.lab", "port": 80, "scheme": "http", "profile": "quick"}) + go func() { + req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/scan", bytes.NewReader(body)) + w := httptest.NewRecorder() + h.Scan(w, req) + }() + + seen := map[string]bool{} + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + var msg Message + if err := conn.ReadJSON(&msg); err != nil { + t.Fatal(err) + } + switch msg.Type { + case "recon_port", "recon_page", "recon_finding": + seen[msg.Type] = true + case "recon_complete": + seen[msg.Type] = true + if !seen["recon_port"] { + t.Fatalf("recon_complete before recon_port; seen=%v", seen) + } + return + } + } + t.Fatalf("missing recon events; seen=%v", seen) +} diff --git a/server/internal/db/recon_scans.go b/server/internal/db/recon_scans.go new file mode 100644 index 0000000..e197c47 --- /dev/null +++ b/server/internal/db/recon_scans.go @@ -0,0 +1,111 @@ +package db + +import ( + "encoding/json" + "fmt" + "time" + + "crypto-miner-server/internal/recon" +) + +func (d *Database) ensureReconScansTable() error { + _, err := d.Exec(`CREATE TABLE IF NOT EXISTS recon_scans ( + scan_id TEXT PRIMARY KEY, + host TEXT NOT NULL, + profile TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'complete', + scanned_at DATETIME NOT NULL, + report_json TEXT NOT NULL + )`) + if err != nil { + return err + } + _, err = d.Exec(`CREATE INDEX IF NOT EXISTS idx_recon_scans_host ON recon_scans(host, scanned_at DESC)`) + return err +} + +func (d *Database) InsertReconScan(report *recon.ScanReport) error { + if d == nil || report == nil || report.ScanID == "" { + return fmt.Errorf("invalid recon scan row") + } + if err := d.ensureReconScansTable(); err != nil { + return err + } + raw, err := json.Marshal(report) + if err != nil { + return err + } + status := report.Status + if status == "" { + status = "complete" + } + _, err = d.Exec(`INSERT INTO recon_scans (scan_id, host, profile, status, scanned_at, report_json) VALUES (?, ?, ?, ?, ?, ?)`, + report.ScanID, report.Host, report.Profile, status, report.ScannedAt.UTC().Format(time.RFC3339Nano), string(raw)) + if err != nil { + return err + } + return d.PruneReconScansForHost(report.Host, 10) +} + +func (d *Database) GetReconScan(scanID string) (*recon.ScanReport, error) { + if err := d.ensureReconScansTable(); err != nil { + return nil, err + } + var host, profile, status, scannedAt, raw string + err := d.QueryRow(`SELECT host, profile, status, scanned_at, report_json FROM recon_scans WHERE scan_id = ?`, scanID).Scan(&host, &profile, &status, &scannedAt, &raw) + if err != nil { + return nil, err + } + var report recon.ScanReport + if err := json.Unmarshal([]byte(raw), &report); err != nil { + return nil, err + } + if report.ScanID == "" { + report.ScanID = scanID + } + if report.Host == "" { + report.Host = host + } + if report.Profile == "" { + report.Profile = profile + } + if report.Status == "" { + report.Status = status + } + return &report, nil +} + +func (d *Database) ListReconScansByHost(host string, limit int) ([]*recon.ScanReport, error) { + if err := d.ensureReconScansTable(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 10 + } + rows, err := d.Query(`SELECT report_json FROM recon_scans WHERE host = ? ORDER BY scanned_at ASC LIMIT ?`, host, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*recon.ScanReport + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var report recon.ScanReport + if err := json.Unmarshal([]byte(raw), &report); err != nil { + return nil, err + } + out = append(out, &report) + } + return out, rows.Err() +} + +func (d *Database) PruneReconScansForHost(host string, keep int) error { + if keep <= 0 { + keep = 10 + } + _, err := d.Exec(`DELETE FROM recon_scans WHERE host = ? AND scan_id NOT IN (SELECT scan_id FROM recon_scans WHERE host = ? ORDER BY scanned_at DESC LIMIT ?)`, host, host, keep) + return err +} diff --git a/server/internal/db/sqlite.go b/server/internal/db/sqlite.go index 29a9047..a552929 100644 --- a/server/internal/db/sqlite.go +++ b/server/internal/db/sqlite.go @@ -279,6 +279,9 @@ func (d *Database) migrate() error { if err := d.ensureSeerTables(); err != nil { return fmt.Errorf("seer migration: %w", err) } + if err := d.ensureReconScansTable(); err != nil { + return fmt.Errorf("recon_scans migration: %w", err) + } if err := d.ensureSubnetDiscoveriesTable(); err != nil { return fmt.Errorf("subnet_discoveries migration: %w", err) } diff --git a/server/internal/recon/recon_stream_test.go b/server/internal/recon/recon_stream_test.go new file mode 100644 index 0000000..389d901 --- /dev/null +++ b/server/internal/recon/recon_stream_test.go @@ -0,0 +1,98 @@ +package recon + +import ( + "testing" + "time" +) + +func TestProfileOptions(t *testing.T) { + quick := ProfileOptions(ProfileQuick) + if len(quick.Ports) != 10 || quick.MaxPages != 1 || quick.SkipPorts { + t.Fatalf("quick=%+v", quick) + } + deep := ProfileOptions(ProfileDeep) + if deep.MaxPages != 50 || deep.CrawlDepth != 2 { + t.Fatalf("deep=%+v", deep) + } + ssrf := ProfileOptions(ProfileSSRFOnly) + if !ssrf.SkipPorts || ssrf.MaxPages != 50 { + t.Fatalf("ssrf=%+v", ssrf) + } +} + +func TestScanStreamEmitsPortsFirst(t *testing.T) { + SetPortDialHook(func(host string, port int, _ time.Duration) bool { + return port == 80 || port == 443 + }) + t.Cleanup(func() { SetPortDialHook(nil) }) + SetFetchPageHook(func(rawURL string) (int, string, error) { + return 200, `Home`, nil + }) + t.Cleanup(func() { SetFetchPageHook(nil) }) + + var events []string + report, err := ScanStream(ScanRequest{Host: "stream.lab", Profile: ProfileQuick, Port: 80, Scheme: "http"}, "scan-1", func(eventType string, _ map[string]interface{}) { + events = append(events, eventType) + }) + if err != nil { + t.Fatal(err) + } + if report.ScanID != "scan-1" || report.Profile != ProfileQuick { + t.Fatalf("report=%+v", report) + } + if len(events) == 0 { + t.Fatal("expected streaming events") + } + firstPort := -1 + for i, e := range events { + if e == "recon_port" && firstPort < 0 { + firstPort = i + } + } + if firstPort < 0 { + t.Fatalf("missing recon_port events: %v", events) + } + for i, e := range events { + if e == "recon_page" && i < firstPort { + t.Fatalf("pages before ports: %v", events) + } + } +} + +func TestDiffReportsNewPortsAndForms(t *testing.T) { + prev := &ScanReport{ + Ports: []PortResult{{Port: 80, Open: true}}, + Crawl: &CrawlReport{MultipartForms: []FormFinding{{PageURL: "http://a/", Action: "/upload"}}}, + } + cur := &ScanReport{ + Ports: []PortResult{{Port: 80, Open: true}, {Port: 443, Open: true}}, + Crawl: &CrawlReport{ + MultipartForms: []FormFinding{ + {PageURL: "http://a/", Action: "/upload"}, + {PageURL: "http://a/admin", Action: "/post"}, + }, + }, + } + diff := DiffReports(prev, cur) + if len(diff.NewPorts) != 1 || diff.NewPorts[0] != 443 { + t.Fatalf("ports=%v", diff.NewPorts) + } + if len(diff.NewForms) != 1 || diff.NewForms[0].Action != "/post" { + t.Fatalf("forms=%v", diff.NewForms) + } +} + +func TestBuildHistoryDiffChain(t *testing.T) { + now := time.Now().UTC() + rows := []*ScanReport{ + {ScanID: "a", Host: "h", ScannedAt: now, Ports: []PortResult{{Port: 22, Open: true}}}, + {ScanID: "b", Host: "h", ScannedAt: now.Add(time.Minute), Ports: []PortResult{{Port: 22, Open: true}, {Port: 80, Open: true}}}, + } + hist := BuildHistory(rows) + if len(hist) != 2 || hist[0].Diff == nil || len(hist[0].Diff.NewPorts) != 1 { + t.Fatalf("first diff=%+v", hist[0].Diff) + } + if hist[1].Diff == nil || len(hist[1].Diff.NewPorts) != 1 || hist[1].Diff.NewPorts[0] != 80 { + t.Fatalf("second diff=%+v", hist[1].Diff) + } +}