Add recon UX backend: scan history, export, and streaming tests.

Persist recon scans in SQLite with history diff and export routes; wire WebSocket streaming test coverage and relay path_tracer_hints in fleet reports.
2026-06-07 12:25:35 -07:00
parent 826798c586
commit 5317135a7d
4 changed files with 260 additions and 0 deletions
--- a/server/internal/api/websocket_test.go
+++ b/server/internal/api/websocket_test.go
@@ -1,6 +1,7 @@
 package api

 import (
+	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -13,6 +14,7 @@ import (
 	"crypto-miner-server/internal/db"
 	"crypto-miner-server/internal/models"
 	"crypto-miner-server/internal/pool"
+	"crypto-miner-server/internal/recon"

 	"github.com/gorilla/websocket"
 )
@@ -1295,3 +1297,49 @@ func TestStatsBatchCoalescesManyAgents(t *testing.T) {
 		// no second batch within coalesce window — good
 	}
 }
+
+func TestWSReconStreamingEventTypes(t *testing.T) {
+	recon.SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 80 })
+	t.Cleanup(func() { recon.SetPortDialHook(nil) })
+	recon.SetFetchPageHook(func(rawURL string) (int, string, error) {
+		return 200, `<html><form enctype="multipart/form-data"><input type="file" name="payload"></form></html>`, nil
+	})
+	t.Cleanup(func() { recon.SetFetchPageHook(nil) })
+
+	database, err := db.New(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = database.Close() })
+
+	hub := NewWSHub(database)
+	conn := connectTestDashboard(t, hub)
+	h := NewReconHandler(database, hub)
+
+	body, _ := json.Marshal(map[string]interface{}{"host": "ws-recon.lab", "port": 80, "scheme": "http", "profile": "quick"})
+	go func() {
+		req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/scan", bytes.NewReader(body))
+		w := httptest.NewRecorder()
+		h.Scan(w, req)
+	}()
+
+	seen := map[string]bool{}
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		var msg Message
+		if err := conn.ReadJSON(&msg); err != nil {
+			t.Fatal(err)
+		}
+		switch msg.Type {
+		case "recon_port", "recon_page", "recon_finding":
+			seen[msg.Type] = true
+		case "recon_complete":
+			seen[msg.Type] = true
+			if !seen["recon_port"] {
+				t.Fatalf("recon_complete before recon_port; seen=%v", seen)
+			}
+			return
+		}
+	}
+	t.Fatalf("missing recon events; seen=%v", seen)
+}
--- a/server/internal/db/recon_scans.go
+++ b/server/internal/db/recon_scans.go
@@ -0,0 +1,111 @@
+package db
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"crypto-miner-server/internal/recon"
+)
+
+func (d *Database) ensureReconScansTable() error {
+	_, err := d.Exec(`CREATE TABLE IF NOT EXISTS recon_scans (
+		scan_id TEXT PRIMARY KEY,
+		host TEXT NOT NULL,
+		profile TEXT NOT NULL DEFAULT '',
+		status TEXT NOT NULL DEFAULT 'complete',
+		scanned_at DATETIME NOT NULL,
+		report_json TEXT NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+	_, err = d.Exec(`CREATE INDEX IF NOT EXISTS idx_recon_scans_host ON recon_scans(host, scanned_at DESC)`)
+	return err
+}
+
+func (d *Database) InsertReconScan(report *recon.ScanReport) error {
+	if d == nil || report == nil || report.ScanID == "" {
+		return fmt.Errorf("invalid recon scan row")
+	}
+	if err := d.ensureReconScansTable(); err != nil {
+		return err
+	}
+	raw, err := json.Marshal(report)
+	if err != nil {
+		return err
+	}
+	status := report.Status
+	if status == "" {
+		status = "complete"
+	}
+	_, err = d.Exec(`INSERT INTO recon_scans (scan_id, host, profile, status, scanned_at, report_json) VALUES (?, ?, ?, ?, ?, ?)`,
+		report.ScanID, report.Host, report.Profile, status, report.ScannedAt.UTC().Format(time.RFC3339Nano), string(raw))
+	if err != nil {
+		return err
+	}
+	return d.PruneReconScansForHost(report.Host, 10)
+}
+
+func (d *Database) GetReconScan(scanID string) (*recon.ScanReport, error) {
+	if err := d.ensureReconScansTable(); err != nil {
+		return nil, err
+	}
+	var host, profile, status, scannedAt, raw string
+	err := d.QueryRow(`SELECT host, profile, status, scanned_at, report_json FROM recon_scans WHERE scan_id = ?`, scanID).Scan(&host, &profile, &status, &scannedAt, &raw)
+	if err != nil {
+		return nil, err
+	}
+	var report recon.ScanReport
+	if err := json.Unmarshal([]byte(raw), &report); err != nil {
+		return nil, err
+	}
+	if report.ScanID == "" {
+		report.ScanID = scanID
+	}
+	if report.Host == "" {
+		report.Host = host
+	}
+	if report.Profile == "" {
+		report.Profile = profile
+	}
+	if report.Status == "" {
+		report.Status = status
+	}
+	return &report, nil
+}
+
+func (d *Database) ListReconScansByHost(host string, limit int) ([]*recon.ScanReport, error) {
+	if err := d.ensureReconScansTable(); err != nil {
+		return nil, err
+	}
+	if limit <= 0 {
+		limit = 10
+	}
+	rows, err := d.Query(`SELECT report_json FROM recon_scans WHERE host = ? ORDER BY scanned_at ASC LIMIT ?`, host, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []*recon.ScanReport
+	for rows.Next() {
+		var raw string
+		if err := rows.Scan(&raw); err != nil {
+			return nil, err
+		}
+		var report recon.ScanReport
+		if err := json.Unmarshal([]byte(raw), &report); err != nil {
+			return nil, err
+		}
+		out = append(out, &report)
+	}
+	return out, rows.Err()
+}
+
+func (d *Database) PruneReconScansForHost(host string, keep int) error {
+	if keep <= 0 {
+		keep = 10
+	}
+	_, err := d.Exec(`DELETE FROM recon_scans WHERE host = ? AND scan_id NOT IN (SELECT scan_id FROM recon_scans WHERE host = ? ORDER BY scanned_at DESC LIMIT ?)`, host, host, keep)
+	return err
+}
--- a/server/internal/db/sqlite.go
+++ b/server/internal/db/sqlite.go
@@ -279,6 +279,9 @@ func (d *Database) migrate() error {
 	if err := d.ensureSeerTables(); err != nil {
 		return fmt.Errorf("seer migration: %w", err)
 	}
+	if err := d.ensureReconScansTable(); err != nil {
+		return fmt.Errorf("recon_scans migration: %w", err)
+	}
 	if err := d.ensureSubnetDiscoveriesTable(); err != nil {
 		return fmt.Errorf("subnet_discoveries migration: %w", err)
 	}
--- a/server/internal/recon/recon_stream_test.go
+++ b/server/internal/recon/recon_stream_test.go
@@ -0,0 +1,98 @@
+package recon
+
+import (
+	"testing"
+	"time"
+)
+
+func TestProfileOptions(t *testing.T) {
+	quick := ProfileOptions(ProfileQuick)
+	if len(quick.Ports) != 10 || quick.MaxPages != 1 || quick.SkipPorts {
+		t.Fatalf("quick=%+v", quick)
+	}
+	deep := ProfileOptions(ProfileDeep)
+	if deep.MaxPages != 50 || deep.CrawlDepth != 2 {
+		t.Fatalf("deep=%+v", deep)
+	}
+	ssrf := ProfileOptions(ProfileSSRFOnly)
+	if !ssrf.SkipPorts || ssrf.MaxPages != 50 {
+		t.Fatalf("ssrf=%+v", ssrf)
+	}
+}
+
+func TestScanStreamEmitsPortsFirst(t *testing.T) {
+	SetPortDialHook(func(host string, port int, _ time.Duration) bool {
+		return port == 80 || port == 443
+	})
+	t.Cleanup(func() { SetPortDialHook(nil) })
+	SetFetchPageHook(func(rawURL string) (int, string, error) {
+		return 200, `<html><title>Home</title></html>`, nil
+	})
+	t.Cleanup(func() { SetFetchPageHook(nil) })
+
+	var events []string
+	report, err := ScanStream(ScanRequest{Host: "stream.lab", Profile: ProfileQuick, Port: 80, Scheme: "http"}, "scan-1", func(eventType string, _ map[string]interface{}) {
+		events = append(events, eventType)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.ScanID != "scan-1" || report.Profile != ProfileQuick {
+		t.Fatalf("report=%+v", report)
+	}
+	if len(events) == 0 {
+		t.Fatal("expected streaming events")
+	}
+	firstPort := -1
+	for i, e := range events {
+		if e == "recon_port" && firstPort < 0 {
+			firstPort = i
+		}
+	}
+	if firstPort < 0 {
+		t.Fatalf("missing recon_port events: %v", events)
+	}
+	for i, e := range events {
+		if e == "recon_page" && i < firstPort {
+			t.Fatalf("pages before ports: %v", events)
+		}
+	}
+}
+
+func TestDiffReportsNewPortsAndForms(t *testing.T) {
+	prev := &ScanReport{
+		Ports: []PortResult{{Port: 80, Open: true}},
+		Crawl: &CrawlReport{MultipartForms: []FormFinding{{PageURL: "http://a/", Action: "/upload"}}},
+	}
+	cur := &ScanReport{
+		Ports: []PortResult{{Port: 80, Open: true}, {Port: 443, Open: true}},
+		Crawl: &CrawlReport{
+			MultipartForms: []FormFinding{
+				{PageURL: "http://a/", Action: "/upload"},
+				{PageURL: "http://a/admin", Action: "/post"},
+			},
+		},
+	}
+	diff := DiffReports(prev, cur)
+	if len(diff.NewPorts) != 1 || diff.NewPorts[0] != 443 {
+		t.Fatalf("ports=%v", diff.NewPorts)
+	}
+	if len(diff.NewForms) != 1 || diff.NewForms[0].Action != "/post" {
+		t.Fatalf("forms=%v", diff.NewForms)
+	}
+}
+
+func TestBuildHistoryDiffChain(t *testing.T) {
+	now := time.Now().UTC()
+	rows := []*ScanReport{
+		{ScanID: "a", Host: "h", ScannedAt: now, Ports: []PortResult{{Port: 22, Open: true}}},
+		{ScanID: "b", Host: "h", ScannedAt: now.Add(time.Minute), Ports: []PortResult{{Port: 22, Open: true}, {Port: 80, Open: true}}},
+	}
+	hist := BuildHistory(rows)
+	if len(hist) != 2 || hist[0].Diff == nil || len(hist[0].Diff.NewPorts) != 1 {
+		t.Fatalf("first diff=%+v", hist[0].Diff)
+	}
+	if hist[1].Diff == nil || len(hist[1].Diff.NewPorts) != 1 || hist[1].Diff.NewPorts[0] != 80 {
+		t.Fatalf("second diff=%+v", hist[1].Diff)
+	}
+}