diff --git a/.gitignore b/.gitignore index cd21efb..7dc4ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,27 @@ _*.txt # Local APK / test logs (not tracked) /agent-tablet-1.apk /server/web/test-output.txt + +# Go build cache (local) +.gocache/ + +# Local tools binaries +/tools/ + +# Agent one-off landing scripts (never commit) +scripts/_ship*.py +scripts/batch*.py +scripts/fix_recon*.py +scripts/install_*.py +scripts/ship_*.py +scripts/write-*.py +scripts/deploy-recon-ux.py +scripts/run-recon-commit.py +scripts/commit-recon*.py +scripts/_batch*.py +scripts/_final_patch.py +scripts/_update_docs.py +server/_deploy_recon*.py +server/patch_recon.py +server/web/_write*.py +server/web/patch_uihelp.py diff --git a/scripts/_batch1_embed.py b/scripts/_batch1_embed.py deleted file mode 100644 index 0a13dc6..0000000 --- a/scripts/_batch1_embed.py +++ /dev/null @@ -1,232 +0,0 @@ -TYPES = '''package recon - -import "time" - -var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} - -const ( - DefaultPortDialTimeout = 2 * time.Second - DefaultCrawlDepth = 2 - DefaultCrawlMaxPages = 50 -) - -type ScanRequest struct { - Host string `json:"host"` - Port int `json:"port,omitempty"` - Scheme string `json:"scheme,omitempty"` - Paths []string `json:"paths,omitempty"` - Profile string `json:"profile,omitempty"` - Profiles []string `json:"profiles,omitempty"` -} - -type PortResult struct { - Port int `json:"port"` - Open bool `json:"open"` -} - -type PortBanner struct { - Port int `json:"port"` - Service string `json:"service,omitempty"` - Banner string `json:"banner,omitempty"` - Title string `json:"title,omitempty"` - Hint string `json:"hint,omitempty"` -} - -type StackEntry struct { - Name string `json:"name"` - Source string `json:"source"` - Detail string `json:"detail,omitempty"` -} - -type HTTPHeaderSnap struct { - URL string - Headers map[string]string -} - -type FormFinding struct { - PageURL string `json:"page_url"` - Action string `json:"action,omitempty"` - Method string `json:"method,omitempty"` - Enctype string `json:"enctype,omitempty"` - Fields []string `json:"fields,omitempty"` - HasFile bool `json:"has_file_input,omitempty"` - Multipart bool `json:"multipart,omitempty"` -} - -type URLFieldFinding struct { - PageURL string `json:"page_url"` - Name string `json:"name"` - Type string `json:"type,omitempty"` - Hint string `json:"hint"` -} - -type PageFinding struct { - URL string `json:"url"` - StatusCode int `json:"status_code"` - Title string `json:"title,omitempty"` -} - -type CrawlReport struct { - PagesFetched int `json:"pages_fetched"` - Pages []PageFinding `json:"pages,omitempty"` - FileInputs []FormFinding `json:"file_inputs,omitempty"` - MultipartForms []FormFinding `json:"multipart_forms,omitempty"` - URLFields []URLFieldFinding `json:"url_fields,omitempty"` - SSRFScore int `json:"ssrf_score"` - CMSFingerprints []string `json:"cms_fingerprints,omitempty"` - Stack []StackEntry `json:"stack,omitempty"` -} - -type DeployRecommendation struct { - Lane string `json:"lane,omitempty"` - Template string `json:"template,omitempty"` - Reason string `json:"reason"` - Priority int `json:"priority"` -} - -type ReconScanDiff struct { - NewPorts []int `json:"new_ports,omitempty"` - NewForms []FormFinding `json:"new_forms,omitempty"` -} - -type ReconHistoryEntry struct { - ScanID string `json:"scan_id"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - Status string `json:"status,omitempty"` - ScannedAt time.Time `json:"scanned_at"` - Report *ScanReport `json:"report"` - Diff *ReconScanDiff `json:"diff,omitempty"` -} - -type ScanReport struct { - ScanID string `json:"scan_id,omitempty"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - ProfilesUsed []string `json:"profiles_used,omitempty"` - Status string `json:"status,omitempty"` - ScannedAt time.Time `json:"scanned_at"` - Ports []PortResult `json:"ports"` - Banners []PortBanner `json:"banners,omitempty"` - Stack []StackEntry `json:"stack,omitempty"` - DeployKitLane string `json:"deploy_kit_lane,omitempty"` - Crawl *CrawlReport `json:"crawl,omitempty"` - RelayVia string `json:"relay_via,omitempty"` - Recommendations []DeployRecommendation `json:"recommendations,omitempty"` -} -''' - -PORTSCAN = '''package recon - -import ( - "net" - "sort" - "strconv" - "strings" - "time" -) - -const ( - PortProfileWeb = "web" - PortProfileWindows = "windows" - PortProfileLinux = "linux" - PortProfileCloudMetadata = "cloud_metadata" -) - -var portProfilePorts = map[string][]int{ - PortProfileWeb: {80, 443, 6262}, - PortProfileWindows: {445, 5985, 3389}, - PortProfileLinux: {22}, -} - -var dialPortFn func(host string, port int, timeout time.Duration) bool - -func ScanPorts(host string, ports []int) []PortResult { - if len(ports) == 0 { - ports = FleetPorts - } - out := make([]PortResult, 0, len(ports)) - for _, port := range ports { - out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) - } - return out -} - -func dialPort(host string, port int, timeout time.Duration) bool { - if dialPortFn != nil { - return dialPortFn(host, port, timeout) - } - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout) - if err != nil { - return false - } - _ = conn.Close() - return true -} - -func ResolveScanPortsFromRequest(req ScanRequest) ([]int, []string) { - return ResolveScanPorts(req.Profiles) -} - -func ResolveScanPorts(profiles []string) ([]int, []string) { - seen := map[int]bool{} - var ports []int - for _, p := range FleetPorts { - if !seen[p] { - seen[p] = true - ports = append(ports, p) - } - } - var used []string - for _, raw := range profiles { - name := strings.ToLower(strings.TrimSpace(raw)) - if name == "" { - continue - } - if name == PortProfileCloudMetadata { - if !containsPortProfile(used, name) { - used = append(used, name) - } - continue - } - bundle, ok := portProfilePorts[name] - if !ok { - continue - } - if !containsPortProfile(used, name) { - used = append(used, name) - } - for _, p := range bundle { - if !seen[p] { - seen[p] = true - ports = append(ports, p) - } - } - } - sort.Ints(ports) - return ports, used -} - -func containsPortProfile(list []string, want string) bool { - for _, s := range list { - if s == want { - return true - } - } - return false -} - -func TargetLooksEC2(host string) bool { - host = strings.ToLower(strings.TrimSpace(host)) - if host == "" { - return false - } - if host == "169.254.169.254" { - return true - } - return strings.Contains(host, ".compute.amazonaws.com") || - strings.Contains(host, ".compute.internal") || - strings.HasPrefix(host, "ip-10-") || - strings.HasPrefix(host, "ec2-") -} -''' diff --git a/scripts/_batch1_scan.go b/scripts/_batch1_scan.go deleted file mode 100644 index 5ac5e00..0000000 --- a/scripts/_batch1_scan.go +++ /dev/null @@ -1,367 +0,0 @@ -package recon - -import ( - "bufio" - "fmt" - "io" - "net" - "net/http" - "regexp" - "strconv" - "strings" - "time" -) - -// Scan runs port scan and optional web crawl for an operator-supplied owned target. -func Scan(req ScanRequest) (*ScanReport, error) { - return runOwnedTargetScan(req, "", nil) -} - -func scanLegacy(req ScanRequest) (*ScanReport, error) { - host, err := normalizeOwnedHost(req.Host) - if err != nil { - return nil, err - } - ports := ScanPorts(host, nil) - report := &ScanReport{ - Host: host, - ScannedAt: time.Now().UTC(), - Ports: ports, - } - - if shouldCrawl(req, ports) { - crawl, err := Crawl(host, req.Port, req.Scheme, req.Paths) - if err == nil && crawl != nil { - report.Crawl = crawl - } - } - report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false) - return report, nil -} - -func normalizeOwnedHost(host string) (string, error) { - host = strings.TrimSpace(host) - if host == "" { - return "", fmt.Errorf("host required") - } - host = strings.TrimPrefix(host, "http://") - host = strings.TrimPrefix(host, "https://") - if i := strings.Index(host, "/"); i >= 0 { - host = host[:i] - } - if h, p, err := net.SplitHostPort(host); err == nil { - if strings.TrimSpace(h) == "" { - return "", fmt.Errorf("invalid host") - } - _ = p - host = h - } - if ip := net.ParseIP(host); ip != nil { - return ip.String(), nil - } - if len(host) > 253 || strings.Contains(host, " ") { - return "", fmt.Errorf("invalid host") - } - return strings.ToLower(host), nil -} - -func shouldCrawl(req ScanRequest, ports []PortResult) bool { - if req.Port > 0 || strings.TrimSpace(req.Scheme) != "" || len(req.Paths) > 0 { - return true - } - for _, p := range ports { - switch p.Port { - case 80, 443, 6262, 8080, 8443: - if p.Open { - return true - } - } - } - return false -} - -// BuildRecommendations maps port and crawl findings to existing deploy lanes/templates. -func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation { - var recs []DeployRecommendation - open := map[int]bool{} - for _, p := range ports { - if p.Open { - open[p.Port] = true - } - } - - if open[22] { - recs = append(recs, DeployRecommendation{ - Lane: "linux_lotl", - Template: "linux-lotl", - Reason: "TCP 22 open — SSH LOTL bootstrap", - Priority: 25, - }) - } - if open[445] { - recs = append(recs, DeployRecommendation{ - Lane: "spread_smb_unc", - Reason: "TCP 445 open — SMB UNC spread", - Priority: 50, - }) - } - if open[5985] || open[5986] { - recs = append(recs, DeployRecommendation{ - Lane: "winrm", - Template: "winrm", - Reason: "TCP 5985/5986 open — WinRM bootstrap", - Priority: 30, - }) - } - if open[80] || open[443] || open[8080] || open[8443] { - recs = append(recs, DeployRecommendation{ - Lane: "bits_curl", - Reason: "HTTP surface open — dropper curl|bash one-liner", - Priority: 20, - }) - recs = append(recs, DeployRecommendation{ - Template: "public_waterhole", - Reason: "HTTP surface open — copy /spread/ public waterhole landing", - Priority: 15, - }) - } - - if crawl != nil { - if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 { - recs = append(recs, DeployRecommendation{ - Lane: "stage_fetch", - Reason: "Multipart or file-upload form — stage_fetch manifest staging", - Priority: 35, - }) - } - if crawl.SSRFScore >= 30 { - recs = append(recs, DeployRecommendation{ - Template: "ssrf_probe", - Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore), - Priority: 45, - }) - } - } - - if lane := SuggestDeployKitLane(stack); lane != "" { - recs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf("Technology stack suggests %s deploy-kit lane", lane), Priority: 18}) - } - if cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) { - recs = append(recs, DeployRecommendation{Lane: "ssm_document", Template: "ssm_document", Reason: "cloud_metadata profile - EC2/IMDS reachable; SSM document lane", Priority: 40}) - } - return dedupeRecommendations(recs) -} - -func dedupeRecommendations(in []DeployRecommendation) []DeployRecommendation { - seen := map[string]bool{} - var out []DeployRecommendation - for _, r := range in { - key := r.Lane + "|" + r.Template - if seen[key] { - continue - } - seen[key] = true - out = append(out, r) - } - return out -} - -func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) } - -func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { - host, err := normalizeOwnedHost(req.Host) - if err != nil { - return nil, err - } - uxProfile := NormalizeProfile(req.Profile) - opts := ProfileOptions(uxProfile) - portsToScan, profilesUsed := ResolveScanPortsFromRequest(req) - if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 { - portsToScan = portsToScan[:10] - } - report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()} - if !opts.SkipPorts { - ports := scanPortsList(host, portsToScan) - report.Ports = ports - report.Banners = GrabBanners(host, ports) - } - if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { - crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit) - if err == nil && crawl != nil { - report.Crawl = crawl - report.Stack = MergeStack(crawl.Stack) - } - } - report.DeployKitLane = SuggestDeployKitLane(report.Stack) - cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) - report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta) - return report, nil -} - -const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only") -type StreamEmit func(eventType string, payload map[string]interface{}) -type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool } - -func NormalizeProfile(profile string) string { - switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" } -} -func ProfileOptions(profile string) ScanOptions { - switch NormalizeProfile(profile) { - case ProfileQuick: - p := FleetPorts - if len(p) > 10 { p = p[:10] } - return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0} - case ProfileDeep: - return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2} - case ProfileSSRFOnly: - return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2} - default: - return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth} - } -} -func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) } -func scanPortsList(host string, ports []int) []PortResult { - if len(ports) == 0 { return ScanPorts(host, nil) } - out := make([]PortResult, 0, len(ports)) - for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } - return out -} -func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool { - if opts.SkipCrawl { return false } - if profile == ProfileSSRFOnly { return true } - return shouldCrawl(req, ports) -} - -func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) { - scheme = normalizeScheme(scheme, port) - if port <= 0 { port = defaultPortForScheme(scheme) } - base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) - seeds := seedPaths - if len(seeds) == 0 { seeds = []string{"/"} } - maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth - if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } - if maxDepth < 0 { maxDepth = DefaultCrawlDepth } - report := &CrawlReport{} - var headerSnaps []HTTPHeaderSnap - var htmlBodies []string - visited := map[string]bool{} - queue := []queuedURL{} - for _, p := range seeds { - if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) } - } - for len(queue) > 0 && report.PagesFetched < maxPages { - item := queue[0]; queue = queue[1:] - key := normalizeURLKey(item.url) - if visited[key] { continue } - visited[key] = true - status, body, headers, err := fetchPage(item.url) - if err != nil { continue } - report.PagesFetched++ - title, _ := htmlParseTitle(body) - report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) - if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } - htmlBodies = append(htmlBodies, body) - files, multi, fields, pageScore, cms := ParseHTML(item.url, body) - report.FileInputs = append(report.FileInputs, files...) - report.MultipartForms = append(report.MultipartForms, multi...) - report.URLFields = append(report.URLFields, fields...) - report.SSRFScore += pageScore - report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) - if item.depth >= maxDepth { continue } - for _, link := range extractLinks(body) { - abs, err := resolveSameOrigin(base, link) - if err != nil || !sameOrigin(base, abs) { continue } - if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) } - } - } - if report.SSRFScore > 100 { report.SSRFScore = 100 } - report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) - report.Stack = BuildStack(headerSnaps, htmlBodies) - return report, nil -} - -func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } -func DiffReports(prev, cur *ScanReport) *ReconScanDiff { - if cur == nil { return nil } - d := &ReconScanDiff{} - if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d } - po := map[int]bool{} - for _, p := range prev.Ports { if p.Open { po[p.Port] = true } } - for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } } - return d -} -func BuildHistory(rows []*ScanReport) []ReconHistoryEntry { - out := make([]ReconHistoryEntry, 0, len(rows)) - var prev *ScanReport - for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r } - return out -} -func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") } - -var metaGeneratorRe = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]+content=["']([^"']+)["']`) -var bannerTitleRe = regexp.MustCompile(`(?is)]*>(.*?)`) -var probeSSHBannerFn func(host string, port int) string -var probeHTTPTitleFn func(host string, port int) (string, string) -var probeWinRMHintFn func(host string, port int) string -var probeEC2MetaFn func() bool - -func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) { - probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta -} -func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry { - seen := map[string]bool{} - var out []StackEntry - add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } } - for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } } - for _, body := range htmlBodies { - if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") } - if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") } - } - return out -} -func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out } -func SuggestDeployKitLane(stack []StackEntry) string { - for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } } - if len(stack) > 0 { return "static" } - return "" -} -func GrabBanners(host string, ports []PortResult) []PortBanner { - open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } } - var out []PortBanner - if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } } - for _, port := range []int{80, 443, 6262, 8080} { - if !open[port] { continue } - title, server := probeHTTPBanner(host, port) - if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) } - } - if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } } - return out -} -func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") } -func probeSSHBanner(host string, port int) string { - if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) } - c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) - if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout)) - line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line) -} -func probeHTTPBanner(host string, port int) (string, string) { - if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) } - scheme := "http"; if port == 443 { scheme = "https" } - resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port))) - if err != nil { return "", "" }; defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) - title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) } - return title, strings.TrimSpace(resp.Header.Get("Server")) -} -func probeWinRMHint(host string, port int) string { - if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) } - c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) - if e != nil { return "" }; _ = c.Close(); return "winrm_listening" -} -func ProbeEC2Metadata() bool { - if probeEC2MetaFn != nil { return probeEC2MetaFn() } - c := &http.Client{Timeout: 800 * time.Millisecond} - r, e := c.Get("http://169.254.169.254/latest/meta-data/") - if e != nil { return false }; defer r.Body.Close() - return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized -} diff --git a/scripts/_batch1_scan_base.go b/scripts/_batch1_scan_base.go deleted file mode 100644 index 7b109d1..0000000 Binary files a/scripts/_batch1_scan_base.go and /dev/null differ diff --git a/scripts/_batch1_scan_extra.go b/scripts/_batch1_scan_extra.go deleted file mode 100644 index d702955..0000000 --- a/scripts/_batch1_scan_extra.go +++ /dev/null @@ -1,200 +0,0 @@ - -func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) } - -func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { - host, err := normalizeOwnedHost(req.Host) - if err != nil { - return nil, err - } - uxProfile := NormalizeProfile(req.Profile) - opts := ProfileOptions(uxProfile) - portsToScan, profilesUsed := ResolveScanPortsFromRequest(req) - if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 { - portsToScan = portsToScan[:10] - } - report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()} - if !opts.SkipPorts { - ports := scanPortsList(host, portsToScan) - report.Ports = ports - report.Banners = GrabBanners(host, ports) - } - if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { - crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit) - if err == nil && crawl != nil { - report.Crawl = crawl - report.Stack = MergeStack(crawl.Stack) - } - } - report.DeployKitLane = SuggestDeployKitLane(report.Stack) - cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) - report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta) - return report, nil -} - -const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only") -type StreamEmit func(eventType string, payload map[string]interface{}) -type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool } - -func NormalizeProfile(profile string) string { - switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" } -} -func ProfileOptions(profile string) ScanOptions { - switch NormalizeProfile(profile) { - case ProfileQuick: - p := FleetPorts - if len(p) > 10 { p = p[:10] } - return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0} - case ProfileDeep: - return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2} - case ProfileSSRFOnly: - return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2} - default: - return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth} - } -} -func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) } -func scanPortsList(host string, ports []int) []PortResult { - if len(ports) == 0 { return ScanPorts(host, nil) } - out := make([]PortResult, 0, len(ports)) - for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) } - return out -} -func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool { - if opts.SkipCrawl { return false } - if profile == ProfileSSRFOnly { return true } - return shouldCrawl(req, ports) -} - -func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) { - scheme = normalizeScheme(scheme, port) - if port <= 0 { port = defaultPortForScheme(scheme) } - base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) - seeds := seedPaths - if len(seeds) == 0 { seeds = []string{"/"} } - maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth - if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } - if maxDepth < 0 { maxDepth = DefaultCrawlDepth } - report := &CrawlReport{} - var headerSnaps []HTTPHeaderSnap - var htmlBodies []string - visited := map[string]bool{} - queue := []queuedURL{} - for _, p := range seeds { - if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) } - } - for len(queue) > 0 && report.PagesFetched < maxPages { - item := queue[0]; queue = queue[1:] - key := normalizeURLKey(item.url) - if visited[key] { continue } - visited[key] = true - status, body, headers, err := fetchPage(item.url) - if err != nil { continue } - report.PagesFetched++ - title, _ := htmlParseTitle(body) - report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) - if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } - htmlBodies = append(htmlBodies, body) - files, multi, fields, pageScore, cms := ParseHTML(item.url, body) - report.FileInputs = append(report.FileInputs, files...) - report.MultipartForms = append(report.MultipartForms, multi...) - report.URLFields = append(report.URLFields, fields...) - report.SSRFScore += pageScore - report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) - if item.depth >= maxDepth { continue } - for _, link := range extractLinks(body) { - abs, err := resolveSameOrigin(base, link) - if err != nil || !sameOrigin(base, abs) { continue } - if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) } - } - } - if report.SSRFScore > 100 { report.SSRFScore = 100 } - report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) - report.Stack = BuildStack(headerSnaps, htmlBodies) - return report, nil -} - -func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } -func DiffReports(prev, cur *ScanReport) *ReconScanDiff { - if cur == nil { return nil } - d := &ReconScanDiff{} - if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d } - po := map[int]bool{} - for _, p := range prev.Ports { if p.Open { po[p.Port] = true } } - for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } } - return d -} -func BuildHistory(rows []*ScanReport) []ReconHistoryEntry { - out := make([]ReconHistoryEntry, 0, len(rows)) - var prev *ScanReport - for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r } - return out -} -func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") } - -var metaGeneratorRe = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]+content=["']([^"']+)["']`) -var bannerTitleRe = regexp.MustCompile(`(?is)]*>(.*?)`) -var probeSSHBannerFn func(host string, port int) string -var probeHTTPTitleFn func(host string, port int) (string, string) -var probeWinRMHintFn func(host string, port int) string -var probeEC2MetaFn func() bool - -func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) { - probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta -} -func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry { - seen := map[string]bool{} - var out []StackEntry - add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } } - for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } } - for _, body := range htmlBodies { - if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") } - if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") } - } - return out -} -func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out } -func SuggestDeployKitLane(stack []StackEntry) string { - for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } } - if len(stack) > 0 { return "static" } - return "" -} -func GrabBanners(host string, ports []PortResult) []PortBanner { - open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } } - var out []PortBanner - if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } } - for _, port := range []int{80, 443, 6262, 8080} { - if !open[port] { continue } - title, server := probeHTTPBanner(host, port) - if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) } - } - if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } } - return out -} -func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") } -func probeSSHBanner(host string, port int) string { - if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) } - c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) - if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout)) - line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line) -} -func probeHTTPBanner(host string, port int) (string, string) { - if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) } - scheme := "http"; if port == 443 { scheme = "https" } - resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port))) - if err != nil { return "", "" }; defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) - title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) } - return title, strings.TrimSpace(resp.Header.Get("Server")) -} -func probeWinRMHint(host string, port int) string { - if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) } - c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout) - if e != nil { return "" }; _ = c.Close(); return "winrm_listening" -} -func ProbeEC2Metadata() bool { - if probeEC2MetaFn != nil { return probeEC2MetaFn() } - c := &http.Client{Timeout: 800 * time.Millisecond} - r, e := c.Get("http://169.254.169.254/latest/meta-data/") - if e != nil { return false }; defer r.Body.Close() - return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized -} diff --git a/scripts/_final_patch.py b/scripts/_final_patch.py deleted file mode 100644 index d3ddc7c..0000000 --- a/scripts/_final_patch.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path -p = Path(r"G:/crypto miner/scripts/test-suite.ps1") -t = p.read_text(encoding="utf-8") -if "[switch]$SubnetRecon," not in t: - t = t.replace("[switch]$Recon,", "[switch]$Recon,\n [switch]$SubnetRecon,") -blocks = """ -if ($SubnetRecon) { - Invoke-Phase \"Subnet recon (LAN sweep + fleet discoveries)\" { - Push-Location (Join-Path $Root \"agent\") - go test ./client/... ./config/... -run SubnetRecon -count=1 - go test ./deploy/... -run SubnetRecon -count=1 - Pop-Location - Push-Location (Join-Path $Root \"server\\web\") - if (-not (Test-Path \"node_modules\")) { npm install --silent } - npm run test -- --run src/help/fleetDiscoveries.test.ts - Pop-Location - } - Exit-FeatureRun \"SubnetRecon\" -} -if ($Recon) { - Invoke-Phase \"Deploy recon (engine + API + UI)\" { - Push-Location (Join-Path $Root \"server\") - go test ./internal/recon/... -count=1 - go test ./internal/api/... -run \"Recon|DeployKit|FleetSpread\" -count=1 - Pop-Location - Push-Location (Join-Path $Root \"server\\web\") - if (-not (Test-Path \"node_modules\")) { npm install --silent } - npm run test -- --run src/help/deployRecon.test.ts src/help/fleetDiscoveries.test.ts src/pages/DeployReconPage.test.tsx src/help/reconRisk.test.ts src/components/Fleet/ReconBadges.test.tsx src/components/Fleet/CrucibleExpandedOps.test.tsx - Pop-Location - } - if (-not $SkipE2E) { - Invoke-Phase \"Deploy recon E2E smoke (Playwright)\" { - $DataDir = Join-Path $env:TEMP (\"aether-recon-e2e-\" + [guid]::NewGuid().ToString(\"n\")) - New-Item -ItemType Directory -Force -Path $DataDir | Out-Null - $E2EUser = if ($env:AETHERFORGE_E2E_USER) { $env:AETHERFORGE_E2E_USER } else { \"testuser\" } - $E2EPass = if ($env:AETHERFORGE_E2E_PASS) { $env:AETHERFORGE_E2E_PASS } else { \"testpass\" } - [System.IO.File]::WriteAllText((Join-Path $DataDir \"users.json\"), (@{ $E2EUser = $E2EPass } | ConvertTo-Json -Compress)) - $ServerExe = Join-Path $Root \"bin\\miner-server.exe\" - if (-not (Test-Path $ServerExe)) { - Push-Location (Join-Path $Root \"server\") - go build -o $ServerExe . - Pop-Location - } - $env:AETHERFORGE_E2E = \"1\" - $proc = Start-Process -FilePath $ServerExe -ArgumentList \"-port\",\"18989\",\"-data\",$DataDir -WorkingDirectory $Root -PassThru -WindowStyle Hidden - try { - $ready = $false - for ($i = 0; $i -lt 30; $i++) { - try { - $r = Invoke-RestMethod \"http://127.0.0.1:18989/api/v1/health\" -TimeoutSec 2 - if ($r.status -eq \"ok\") { $ready = $true; break } - } catch {} - Start-Sleep -Seconds 1 - } - if (-not $ready) { throw \"recon E2E server not healthy on :18989\" } - Push-Location (Join-Path $Root \"server\\web\") - $env:AETHERFORGE_URL = \"http://127.0.0.1:18989\" - npx playwright test e2e/deploy-recon.spec.ts --config playwright.config.ts - if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { throw \"deploy-recon E2E failed (exit $LASTEXITCODE)\" } - Pop-Location - } finally { - if ($proc -and -not $proc.HasExited) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue } - Remove-Item $DataDir -Recurse -Force -ErrorAction SilentlyContinue - } - } - } - Exit-FeatureRun \"Recon\" -} - -""" -# remove old Recon block(s) before phase 1 -import re -t = re.sub(r"\nif \(\$Recon\) \{.*?\n\}\n\n(?=Invoke-Phase \"1/8)", "\n", t, flags=re.S) -if "if ($SubnetRecon)" not in t: - t = t.replace("if ($ReconOnly) {", blocks + "if ($ReconOnly) {") -p.write_text(t, encoding="utf-8") -print("final patch ok") diff --git a/scripts/_update_docs.py b/scripts/_update_docs.py deleted file mode 100644 index a2050a8..0000000 --- a/scripts/_update_docs.py +++ /dev/null @@ -1,38 +0,0 @@ -from pathlib import Path -import re -readme = Path(r"G:/crypto miner/tests/README.md") -text = readme.read_text(encoding="utf-8") -text = text.replace( - "Go server **960** `Test*` · Go agent **670** `Test*` · Vitest **849** tests in **109** files · Playwright **26** tests in **9** spec files.", - "Go server **1001** `Test*` · Go agent **680** `Test*` · Vitest **867** tests in **113** files · Playwright **32** tests in **10** spec files.", -) -if "Subnet recon" not in text: - text = text.replace( - "| **Fleet recon** | `.\\scripts\\test-suite.ps1 -ReconOnly` | Vuln/CVE, cred graph, triple-onion gates, Path Tracer discover, recon Vitest |", - "| **Fleet recon** | `.\\scripts\\test-suite.ps1 -ReconOnly` | Vuln/CVE, cred graph, triple-onion gates, Path Tracer discover, recon Vitest |\n| **Deploy Recon** | `.\\scripts\\test-suite.ps1 -Recon` | `internal/recon`, recon API + deploy-kit, Deploy Recon Vitest + `deploy-recon.spec.ts` E2E (unless `-SkipE2E`) |\n| **Subnet recon** | `.\\scripts\\test-suite.ps1 -SubnetRecon` | Agent subnet_recon packages + `fleetDiscoveries` Vitest |", - ) - # remove duplicate Deploy Recon if added twice - text = text.replace( - "| **Deploy Recon** | `.\\scripts\\test-suite.ps1 -Recon` | `internal/recon`, `/recon/scan` + deploy-kit API, `DeployReconPage` + `deployRecon` Vitest |\n", - "", - ) -if ".\\scripts\\test-suite.ps1 -SubnetRecon" not in text: - text = text.replace( - ".\\scripts\\test-suite.ps1 -ReconOnly", - ".\\scripts\\test-suite.ps1 -Recon\n.\\scripts\\test-suite.ps1 -SubnetRecon\n.\\scripts\\test-suite.ps1 -ReconOnly", - 1, - ) -readme.write_text(text, encoding="utf-8") -problems = Path(r"G:/crypto miner/PROBLEMS.md") -pt = problems.read_text(encoding="utf-8") -pt = pt.replace( - "Regression tables and counts: Go server **901**, agent **657**, Vitest **849**, Playwright **26**", - "Regression tables and counts: Go server **1001**, agent **680**, Vitest **867**, Playwright **32**", -) -if "Deploy Recon port scan" not in pt: - pt = pt.replace( - "## Do not commit", - "| **Deploy Recon port scan / crawl** | TCP port dial and same-origin HTTP crawl execute on the **dashboard host** (Go server), not from fleet agents. Firewall path must allow the server to reach the owned target. |\n| **Deploy Recon SSRF** | UI copies SSRF probe URLs; **no automated form submit** — operator pastes probe URL into owned target fields manually to validate server-side fetch to install.sh. |\n\n## Do not commit", - ) -problems.write_text(pt, encoding="utf-8") -print("done") diff --git a/scripts/batch1_build_scan.py b/scripts/batch1_build_scan.py deleted file mode 100644 index 84d4f24..0000000 --- a/scripts/batch1_build_scan.py +++ /dev/null @@ -1,34 +0,0 @@ -import subprocess -from pathlib import Path - -ROOT = Path(r"G:/crypto miner") -scan = subprocess.check_output( - ["git", "-C", str(ROOT), "show", "HEAD:server/internal/recon/scan.go"], - text=True, -) -scan = scan.replace( - "import (\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n)", - "import (\n\t\"bufio\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)", -) -scan = scan.replace( - "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", - "func Scan(req ScanRequest) (*ScanReport, error) {\n\treturn runOwnedTargetScan(req, \"\", nil)\n}\n\nfunc scanLegacy(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", -) -scan = scan.replace("ports := ScanPorts(host)", "ports := ScanPorts(host, nil)") -scan = scan.replace( - "report.Recommendations = BuildRecommendations(ports, report.Crawl)", - "report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false)", -) -scan = scan.replace( - "func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecommendation {", - "func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation {", -) -scan = scan.replace("case 80, 443, 8080, 8443:", "case 80, 443, 6262, 8080, 8443:") -# extend BuildRecommendations tail -scan = scan.replace( - "\treturn dedupeRecommendations(recs)\n}", - "\tif lane := SuggestDeployKitLane(stack); lane != \"\" {\n\t\trecs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf(\"Technology stack suggests %s deploy-kit lane\", lane), Priority: 18})\n\t}\n\tif cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) {\n\t\trecs = append(recs, DeployRecommendation{Lane: \"ssm_document\", Template: \"ssm_document\", Reason: \"cloud_metadata profile - EC2/IMDS reachable; SSM document lane\", Priority: 40})\n\t}\n\treturn dedupeRecommendations(recs)\n}", -) -extra = Path(__file__).with_name("_batch1_scan_extra.go").read_text(encoding="utf-8") -Path(__file__).with_name("_batch1_scan.go").write_text(scan + extra, encoding="utf-8") -print("built scan", len(scan + extra)) diff --git a/scripts/batch1_commit.py b/scripts/batch1_commit.py deleted file mode 100644 index 9099687..0000000 --- a/scripts/batch1_commit.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -from pathlib import Path - -ROOT = Path(r"G:/crypto miner") -RECON = ROOT / "server/internal/recon" -SCRIPTS = Path(__file__).parent - - -def write(name: str, text: str) -> None: - data = text.encode("utf-8") - if b"\x00" in data: - raise ValueError(f"NUL in {name}") - (RECON / name).write_bytes(data) - - -def patch_crawl() -> None: - p = RECON / "crawl.go" - t = p.read_text(encoding="utf-8") - if "headerSnaps" not in t: - t = t.replace("report := &CrawlReport{}\n\tvisited", "report := &CrawlReport{}\n\tvar headerSnaps []HTTPHeaderSnap\n\tvar htmlBodies []string\n\tvisited") - t = t.replace("status, body, err := fetchPage", "status, body, headers, err := fetchPage") - t = t.replace( - "report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})\n\n\t\tfiles", - "report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})\n\t\tif len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) }\n\t\thtmlBodies = append(htmlBodies, body)\n\n\t\tfiles", - ) - t = t.replace( - "report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treturn report, nil", - "report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treport.Stack = BuildStack(headerSnaps, htmlBodies)\n\treturn report, nil", - ) - if "map[string]string, error" not in t: - t = t.replace("func fetchPage(rawURL string) (int, string, error)", "func fetchPage(rawURL string) (int, string, map[string]string, error)") - t = t.replace("return fetchPageFn(rawURL)", "s,b,e:=fetchPageFn(rawURL); return s,b,nil,e") - t = t.replace("return 0, \"\", err", "return 0, \"\", nil, err") - t = t.replace("return resp.StatusCode, \"\", err", "return resp.StatusCode, \"\", nil, err") - t = t.replace( - "return resp.StatusCode, body, nil\n}", - "headers:=map[string]string{}\n\tfor k,v:=range resp.Header { if len(v)>0 { headers[k]=v[0] } }\n\treturn resp.StatusCode, body, headers, nil\n}", - ) - p.write_bytes(t.encode("utf-8")) - - -def patch_tests() -> None: - p = RECON / "recon_test.go" - t = p.read_text(encoding="utf-8") - t = t.replace('ScanPorts("10.0.0.5")', 'ScanPorts("10.0.0.5", nil)') - t = t.replace('BuildRecommendations(ports, crawl)', 'BuildRecommendations(ports, crawl, nil, "10.0.0.1", false)') - extra = ''' - -func TestResolveScanPortsMergesProfiles(t *testing.T) { - ports, used := ResolveScanPorts([]string{"web", "linux", "cloud_metadata"}) - if !containsInt(ports, 6262) || len(used) != 3 { t.Fatalf("%v %v", ports, used) } -} -func TestBuildStackFromHeaders(t *testing.T) { - stack := BuildStack([]HTTPHeaderSnap{{Headers: map[string]string{"X-Powered-By":"PHP/8.1"}}}, nil) - if SuggestDeployKitLane(stack) != "php" { t.Fatal(stack) } -} -func TestGrabBannersWithHooks(t *testing.T) { - SetBannerHooks(func(_ string,p int) string { if p==22 {return "SSH"}; return "" }, func(_ string,p int)(string,string){ if p==80 {return "t","s"}; return "","" }, func(_ string,p int) string { if p==5985 {return "w"}; return "" }, nil) - t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) - if len(GrabBanners("h", []PortResult{{22,true},{80,true},{5985,true}})) != 3 { t.Fatal() } -} -func TestCloudMetadataProfileSuggestsSSM(t *testing.T) { - SetBannerHooks(nil,nil,nil,func()bool{return true}); t.Cleanup(func(){SetBannerHooks(nil,nil,nil,nil)}) - for _,r := range BuildRecommendations(nil,nil,nil,"ec2.compute.amazonaws.com",true) { if r.Lane=="ssm_document" { return } } - t.Fatal() -} -func containsInt(a []int,w int) bool { for _,n:=range a { if n==w {return true} }; return false } -''' - if "TestResolveScanPortsMergesProfiles" not in t: - t += extra - p.write_bytes(t.encode("utf-8")) - - -def patch_recon_ts() -> None: - p = ROOT / "server/web/src/types/recon.ts" - t = p.read_text(encoding="utf-8") - if "profiles_used" in t: - return - t = t.replace("paths?: string[];\n}", "paths?: string[];\n profile?: string;\n profiles?: string[];\n}") - t = t.replace( - "export interface ReconPortResult {\n port: number;\n open: boolean;\n}\n\nexport interface ReconFormFinding", - "export interface ReconPortResult {\n port: number;\n open: boolean;\n}\n\nexport interface ReconPortBanner {\n port: number;\n service?: string;\n banner?: string;\n title?: string;\n hint?: string;\n}\n\nexport interface ReconStackEntry {\n name: string;\n source: string;\n detail?: string;\n}\n\nexport interface ReconFormFinding", - ) - t = t.replace("cms_fingerprints?: string[];\n}", "cms_fingerprints?: string[];\n stack?: ReconStackEntry[];\n}") - t = t.replace( - "export interface ReconScanReport {\n host: string;", - "export interface ReconScanReport {\n scan_id?: string;\n host: string;\n profile?: string;\n profiles_used?: string[];\n status?: string;", - ) - t = t.replace( - "ports: ReconPortResult[];\n crawl?: ReconCrawlReport;", - "ports: ReconPortResult[];\n banners?: ReconPortBanner[];\n stack?: ReconStackEntry[];\n deploy_kit_lane?: string;\n crawl?: ReconCrawlReport;", - ) - p.write_bytes(t.encode("utf-8")) - - -def patch_subnet_recon() -> None: - p = ROOT / "agent/deploy/subnet_recon.go" - t = p.read_text(encoding="utf-8") - if "SSHBanner" in t: - return - t = t.replace("[]int{80, 443, 8080}", "[]int{80, 443, 6262, 8080}") - t = t.replace( - 'HTTPTitle string `json:"http_title,omitempty"`\n\tStatus', - 'HTTPTitle string `json:"http_title,omitempty"`\n\tSSHBanner string `json:"ssh_banner,omitempty"`\n\tWinRMHint string `json:"winrm_hint,omitempty"`\n\tStatus', - ) - t = t.replace( - "entry.HTTPTitle = title\n\t\t}\n\t\tout = append(out, entry)", - "entry.HTTPTitle = title\n\t\t}\n\t\tif b:=probeSSHBanner(host,open);b!=\"\"{entry.SSHBanner=b}\n\t\tif h:=probeWinRMHint(host,open);h!=\"\"{entry.WinRMHint=h}\n\t\tout = append(out, entry)", - ) - insert = ''' -func probeSSHBanner(host string, openPorts []int) string { - for _, p := range openPorts { if p == 22 { - c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"22"), 2*time.Second) - if err != nil { return "" } - defer c.Close() - b := make([]byte, 256); n, _ := c.Read(b) - return strings.TrimSpace(string(b[:n])) - }} - return "" -} -func probeWinRMHint(host string, openPorts []int) string { - for _, p := range openPorts { if p == 5985 { - c, err := net.DialTimeout("tcp", net.JoinHostPort(host,"5985"), 2*time.Second) - if err == nil { _ = c.Close(); return "winrm_listening" } - }} - return "" -} -''' - t = t.replace("func probeHTTPTitle(host string, openPorts []int) string {", insert + "\nfunc probeHTTPTitle(host string, openPorts []int) string {") - p.write_bytes(t.encode("utf-8")) - - -def main() -> None: - for orphan in ("upload_hunter.go", "admin_surface.go", "recon_ux.go"): - p = RECON / orphan - if p.exists(): - p.unlink() - subprocess.run(["python", str(SCRIPTS / "batch1_build_scan.py")], check=True) - subprocess.run(["git", "checkout", "HEAD", "--", "server/internal/recon/crawl.go", "server/internal/recon/recon_test.go", "server/internal/recon/relay_scan.go"], cwd=str(ROOT), check=True) - exec((SCRIPTS / "_batch1_embed.py").read_text(encoding="utf-8"), globals()) - write("types.go", TYPES) - write("portscan.go", PORTSCAN) - write("scan.go", (SCRIPTS / "_batch1_scan.go").read_text(encoding="utf-8")) - ux = RECON / "recon_ux.go" - if ux.exists(): - ux.unlink() - for orphan in ("upload_hunter.go", "admin_surface.go", "recon_ux.go"): - p = RECON / orphan - if p.exists(): - p.unlink() - patch_crawl() - patch_tests() - patch_recon_ts() - patch_subnet_recon() - relay = RECON / "relay_scan.go" - if relay.exists(): - t = relay.read_text(encoding="utf-8") - t = t.replace('ScanPorts(host)', 'ScanPorts(host, nil)') - t = t.replace('BuildRecommendations(ports, nil)', 'BuildRecommendations(ports, nil, nil, host, false)') - t = t.replace('shell.Recommendations = BuildRecommendations(ports, nil, nil, host, false)', 'shell.Recommendations = BuildRecommendations(ports, nil, nil, shell.Host, false)') - t = t.replace( - "Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia,\n\t\tUDPHints: r.UDPHints, PathTracerHints: r.PathTracerHints, Message: r.Message,\n\t\tRecommendations: r.Recommendations,", - "Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia, Recommendations: r.Recommendations,", - ) - relay.write_bytes(t.encode("utf-8")) - env = {**subprocess.os.environ, "GOCACHE": str(ROOT / ".gocache")} - r = subprocess.run(["go", "test", "./internal/recon/..."], cwd=str(ROOT / "server"), env=env) - if r.returncode != 0: - raise SystemExit(r.returncode) - subprocess.run( - ["git", "add", "server/internal/recon", "server/web/src/types/recon.ts", "agent/deploy/subnet_recon.go", "scripts"], - cwd=str(ROOT), check=True, - ) - subprocess.run( - ["git", "commit", "-m", "Add recon network batch 1: stack banners and smart port profiles.\n\nParse technology stack from crawl headers, grab SSH/HTTP/WinRM banners, merge smart port bundles with FleetPorts, and suggest deploy-kit lane plus SSM for EC2 metadata targets."], - cwd=str(ROOT), check=True, - ) - subprocess.run(["git", "push", "origin", "main"], cwd=str(ROOT), check=True) - print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(ROOT), text=True).strip()) - - -if __name__ == "__main__": - main() diff --git a/scripts/commit-recon-batch2.py b/scripts/commit-recon-batch2.py deleted file mode 100644 index e308307..0000000 --- a/scripts/commit-recon-batch2.py +++ /dev/null @@ -1,789 +0,0 @@ -#!/usr/bin/env python3 -"""Write recon batch 2 files, test, commit, and push in one shot.""" -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - -FILES = {} - -FILES["server/internal/recon/relay_scan.go"] = r'''package recon - -import ( - "encoding/json" - "errors" - "net" - "strconv" - "strings" - "time" -) - -var RelayScanUDPPorts = []int{53, 51820} - -type RelayScanRequest struct { - Host string `json:"host"` - UDPGuess bool `json:"udp_guess,omitempty"` -} - -type UDPHint struct { - Port int `json:"port"` - Open bool `json:"open"` - Service string `json:"service"` -} - -type RelayScanReport struct { - Host string `json:"host"` - ScannedAt time.Time `json:"scanned_at"` - LocalReachable bool `json:"local_reachable"` - ScannedVia string `json:"scanned_via"` - RelayAgentID string `json:"relay_agent_id,omitempty"` - RelayAgentName string `json:"relay_agent_name,omitempty"` - Ports []PortResult `json:"ports"` - UDPHints []UDPHint `json:"udp_hints,omitempty"` - PathTracerHints []string `json:"path_tracer_hints,omitempty"` - Message string `json:"message,omitempty"` - Recommendations []DeployRecommendation `json:"recommendations,omitempty"` -} - -var reachabilityPorts = []int{80, 443, 22, 445, 3389} -var hostReachableFn func(host string) bool -var udpGuessFn func(host string, ports []int) []UDPHint - -func SetHostReachableHook(fn func(host string) bool) { hostReachableFn = fn } -func SetUDPGuesssHook(fn func(host string, ports []int) []UDPHint) { udpGuessFn = fn } - -func HostReachable(host string) bool { - if hostReachableFn != nil { - return hostReachableFn(host) - } - host = strings.TrimSpace(host) - for _, port := range reachabilityPorts { - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), 1200*time.Millisecond) - if err == nil { - conn.Close() - return true - } - if isConnRefused(err) { - return true - } - } - return false -} - -func GuessUDPHints(host string, enabled bool) []UDPHint { - if !enabled { - return nil - } - if udpGuessFn != nil { - return udpGuessFn(host, RelayScanUDPPorts) - } - var out []UDPHint - for _, port := range RelayScanUDPPorts { - out = append(out, UDPHint{Port: port, Open: probeUDPQuick(host, port), Service: udpServiceLabel(port)}) - } - return out -} - -func PathTracerHintsFromUDP(hints []UDPHint) []string { - var out []string - seen := map[string]bool{} - for _, h := range hints { - if h.Open && h.Service != "" && !seen[h.Service] { - seen[h.Service] = true - out = append(out, h.Service) - } - } - return out -} - -func LocalRelayScan(req RelayScanRequest) (*RelayScanReport, error) { - host, err := NormalizeHost(req.Host) - if err != nil { - return nil, err - } - ports := ScanPorts(host) - udp := GuessUDPHints(host, req.UDPGuess) - return &RelayScanReport{ - Host: host, ScannedAt: time.Now().UTC(), LocalReachable: true, ScannedVia: "server", - Ports: ports, UDPHints: udp, PathTracerHints: PathTracerHintsFromUDP(udp), - Recommendations: BuildRecommendations(ports, nil), - }, nil -} - -func MergeAgentRelayScan(shell RelayScanReport, agentPayload []byte, udpGuess bool) (*RelayScanReport, error) { - var raw struct { - OpenPorts []int `json:"open_ports"` - UDPHints []UDPHint `json:"udp_hints"` - } - if err := json.Unmarshal(agentPayload, &raw); err != nil { - return nil, err - } - openSet := map[int]bool{} - for _, p := range raw.OpenPorts { - openSet[p] = true - } - ports := make([]PortResult, 0, len(FleetPorts)) - for _, p := range FleetPorts { - ports = append(ports, PortResult{Port: p, Open: openSet[p]}) - } - udp := raw.UDPHints - if udpGuess && len(udp) == 0 { - udp = GuessUDPHints(shell.Host, true) - } - shell.Ports = ports - shell.UDPHints = udp - shell.PathTracerHints = PathTracerHintsFromUDP(udp) - shell.Recommendations = BuildRecommendations(ports, nil) - shell.ScannedAt = time.Now().UTC() - return &shell, nil -} - -func (r *RelayScanReport) ToScanReport() *ScanReport { - if r == nil { - return nil - } - relayVia := "" - if r.ScannedVia == "relay" { - if r.RelayAgentName != "" { - relayVia = r.RelayAgentName - } else if r.RelayAgentID != "" { - relayVia = r.RelayAgentID - } - } - return &ScanReport{ - Host: r.Host, ScannedAt: r.ScannedAt, Ports: r.Ports, RelayVia: relayVia, - UDPHints: r.UDPHints, PathTracerHints: r.PathTracerHints, Message: r.Message, - Recommendations: r.Recommendations, - } -} - -func udpServiceLabel(port int) string { - switch port { - case 53: - return "dns" - case 51820: - return "wireguard" - default: - return "" - } -} - -func isConnRefused(err error) bool { - if err == nil { - return false - } - var opErr *net.OpError - if errors.As(err, &opErr) { - return strings.Contains(strings.ToLower(opErr.Err.Error()), "refused") - } - return strings.Contains(strings.ToLower(err.Error()), "refused") -} - -func probeUDPQuick(host string, port int) bool { - conn, err := net.DialTimeout("udp", net.JoinHostPort(host, strconv.Itoa(port)), 800*time.Millisecond) - if err != nil { - return false - } - defer conn.Close() - _ = conn.SetDeadline(time.Now().Add(800 * time.Millisecond)) - if b := udpProbePayload(port); len(b) > 0 { - _, _ = conn.Write(b) - } - buf := make([]byte, 512) - n, err := conn.Read(buf) - return err == nil && n > 0 -} - -func udpProbePayload(port int) []byte { - switch port { - case 53: - return []byte{0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x07, 'v', 'e', 'r', 's', 'i', 'o', 'n', 0x04, 'b', 'i', 'n', 'd', 0x00, 0x00, 0x10, 0x00, 0x03} - case 51820: - return []byte{0x01, 0x00, 0x00, 0x00} - default: - return nil - } -} -''' - -FILES["server/internal/recon/relay_scan_test.go"] = r'''package recon - -import "testing" - -func TestHostReachableHook(t *testing.T) { - SetHostReachableHook(func(host string) bool { return host == "10.0.0.5" }) - t.Cleanup(func() { SetHostReachableHook(nil) }) - if !HostReachable("10.0.0.5") || HostReachable("10.0.0.6") { - t.Fatal("reachability hook mismatch") - } -} - -func TestMergeAgentRelayScanWireguardHint(t *testing.T) { - payload := []byte(`{"open_ports":[22],"udp_hints":[{"port":51820,"open":true,"service":"wireguard"}]}`) - merged, err := MergeAgentRelayScan(RelayScanReport{Host: "10.1.2.50", ScannedVia: "relay"}, payload, false) - if err != nil || len(merged.PathTracerHints) != 1 || merged.PathTracerHints[0] != "wireguard" { - t.Fatalf("merged=%+v err=%v", merged, err) - } -} -''' - -FILES["server/internal/api/recon_relay_scan.go"] = r'''package api - -import ( - "encoding/json" - "net/http" - "strings" - "time" - - "crypto-miner-server/internal/recon" -) - -const relayScanTimeout = 30 * time.Second - -func (h *ReconHandler) RelayScan(w http.ResponseWriter, r *http.Request) { - var req recon.RelayScanRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, "invalid JSON", http.StatusBadRequest) - return - } - req.Host = strings.TrimSpace(req.Host) - if req.Host == "" { - http.Error(w, "host required", http.StatusBadRequest) - return - } - host, err := recon.NormalizeHost(req.Host) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - req.Host = host - - if recon.HostReachable(host) { - report, err := recon.LocalRelayScan(req) - if err != nil { - writeJSON(w, map[string]interface{}{"ok": false, "error": err.Error()}) - return - } - writeJSON(w, map[string]interface{}{"ok": true, "report": report.ToScanReport()}) - return - } - - if h == nil || h.wsHub == nil { - writeJSON(w, relayScanFallback(host, "", "", "fleet websocket hub unavailable")) - return - } - - seedID, seedName, ok := pickSpreadSeedAgent(h.wsHub, host, "") - if !ok { - writeJSON(w, relayScanFallback(host, "", "", "no online fleet agent on nearby /24 to relay scan for "+host)) - return - } - - ch := h.wsHub.AwaitCommandResult(seedID, "recon_relay_scan") - args := map[string]interface{}{"command": host} - if req.UDPGuess { - args["path"] = "true" - } - if err := h.wsHub.SendAgentCommand(seedID, "recon_relay_scan", args); err != nil { - h.wsHub.CancelAwait(seedID, "recon_relay_scan") - writeJSON(w, map[string]interface{}{ - "ok": false, "agent_id": seedID, "agent_name": seedName, "error": err.Error(), - "report": relayScanFallbackReport(host, seedName, "relay dispatch failed: "+err.Error()), - }) - return - } - - select { - case payload := <-ch: - success, _ := payload["success"].(bool) - msg, _ := payload["message"].(string) - if !success { - writeJSON(w, map[string]interface{}{ - "ok": false, "agent_id": seedID, "agent_name": seedName, "error": strings.TrimSpace(msg), - "report": relayScanFallbackReport(host, seedName, strings.TrimSpace(msg)), - }) - return - } - shell := recon.RelayScanReport{ - Host: host, LocalReachable: false, ScannedVia: "relay", - RelayAgentID: seedID, RelayAgentName: seedName, - } - merged, err := recon.MergeAgentRelayScan(shell, []byte(msg), req.UDPGuess) - if err != nil { - writeJSON(w, map[string]interface{}{ - "ok": false, "agent_id": seedID, "agent_name": seedName, - "error": "invalid agent relay payload: " + err.Error(), - }) - return - } - writeJSON(w, map[string]interface{}{ - "ok": true, "agent_id": seedID, "agent_name": seedName, "report": merged.ToScanReport(), - }) - case <-time.After(relayScanTimeout): - h.wsHub.CancelAwait(seedID, "recon_relay_scan") - writeJSON(w, map[string]interface{}{ - "ok": false, "agent_id": seedID, "agent_name": seedName, "error": "relay scan timed out", - "report": relayScanFallbackReport(host, seedName, "relay scan timed out after 30s"), - }) - } -} - -func relayScanFallback(host, agentID, agentName, message string) map[string]interface{} { - out := map[string]interface{}{"ok": false, "error": message, "report": relayScanFallbackReport(host, agentName, message)} - if agentID != "" { - out["agent_id"] = agentID - } - if agentName != "" { - out["agent_name"] = agentName - } - return out -} - -func relayScanFallbackReport(host, relayVia, message string) *recon.ScanReport { - return &recon.ScanReport{Host: host, ScannedAt: time.Now().UTC(), RelayVia: relayVia, Message: message} -} -''' - -FILES["server/internal/api/recon_relay_scan_test.go"] = r'''package api - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - "time" - - "crypto-miner-server/internal/db" - "crypto-miner-server/internal/recon" -) - -func TestReconRelayScanLocalPath(t *testing.T) { - recon.SetHostReachableHook(func(host string) bool { return true }) - t.Cleanup(func() { recon.SetHostReachableHook(nil) }) - recon.SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 22 }) - t.Cleanup(func() { recon.SetPortDialHook(nil) }) - - h := NewReconHandler(nil, nil) - body, _ := json.Marshal(map[string]interface{}{"host": "10.0.0.10"}) - req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) - w := httptest.NewRecorder() - h.RelayScan(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) - } - var resp map[string]interface{} - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if resp["ok"] != true { - t.Fatalf("resp=%v", resp) - } -} - -func TestReconRelayScanNoRelayFallback(t *testing.T) { - recon.SetHostReachableHook(func(host string) bool { return false }) - t.Cleanup(func() { recon.SetHostReachableHook(nil) }) - - h := NewReconHandler(nil, NewWSHub(nil)) - body, _ := json.Marshal(map[string]interface{}{"host": "10.99.1.50"}) - req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) - w := httptest.NewRecorder() - h.RelayScan(w, req) - var resp map[string]interface{} - _ = json.Unmarshal(w.Body.Bytes(), &resp) - if resp["ok"] != false || resp["error"] == nil { - t.Fatalf("resp=%v", resp) - } -} - -func TestReconRelayScanFleetRelay(t *testing.T) { - recon.SetHostReachableHook(func(host string) bool { return false }) - t.Cleanup(func() { recon.SetHostReachableHook(nil) }) - - database, err := db.New(t.TempDir()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = database.Close() }) - - hub := NewWSHub(database) - agentID := "relay-seed" - conn := connectTestAgentWithIP(t, hub, agentID, "10.42.1.50") - - done := make(chan struct{}) - go func() { - defer close(done) - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - var msg Message - if err := conn.ReadJSON(&msg); err != nil { - return - } - if msg.Type != "command" { - continue - } - var payload map[string]interface{} - if err := json.Unmarshal(msg.Payload, &payload); err != nil { - return - } - if payload["action"] != "recon_relay_scan" { - continue - } - result := `{"open_ports":[22,445],"udp_hints":[{"port":53,"open":true,"service":"dns"}]}` - cmdPayload, _ := json.Marshal(map[string]interface{}{ - "action": "recon_relay_scan", "success": true, "message": result, - }) - _ = conn.WriteJSON(Message{Type: "command_result", Payload: cmdPayload}) - return - } - }() - - h := NewReconHandler(database, hub) - body, _ := json.Marshal(map[string]interface{}{"host": "10.42.1.100", "udp_guess": true}) - req := httptest.NewRequest(http.MethodPost, "/api/v1/recon/relay-scan", bytes.NewReader(body)) - w := httptest.NewRecorder() - h.RelayScan(w, req) - - select { - case <-done: - case <-time.After(6 * time.Second): - t.Fatal("agent did not receive relay command") - } - - var resp struct { - OK bool `json:"ok"` - AgentID string `json:"agent_id"` - Report map[string]interface{} `json:"report"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if !resp.OK || resp.AgentID != agentID { - t.Fatalf("resp=%+v", resp) - } - hints, _ := resp.Report["path_tracer_hints"].([]interface{}) - if len(hints) != 1 || hints[0] != "dns" { - t.Fatalf("report=%v", resp.Report) - } -} -''' - -FILES["agent/client/recon_relay_scan_test.go"] = r'''package client - -import ( - "encoding/json" - "testing" - - "crypto-miner-agent/deploy" -) - -func TestHandleReconRelayScanCommand(t *testing.T) { - deploy.SetProbePortsHook(func(host string, ports []int) []int { - if host == "10.1.2.3" { - return []int{22, 445} - } - return nil - }) - t.Cleanup(func() { deploy.SetProbePortsHook(nil) }) - deploy.SetRelayUDPGuessHook(func(host string, ports []int) []deploy.RelayUDPHint { - return []deploy.RelayUDPHint{{Port: 51820, Open: true, Service: "wireguard"}} - }) - t.Cleanup(func() { deploy.SetRelayUDPGuessHook(nil) }) - - var gotAction string - var gotSuccess bool - var gotMsg string - c := &AgentClient{} - c.commandResultHook = func(action string, success bool, message string) { - gotAction, gotSuccess, gotMsg = action, success, message - } - - if !c.handleReconCommand("recon_relay_scan", "10.1.2.3", "true") { - t.Fatal("expected handled") - } - if gotAction != "recon_relay_scan" || !gotSuccess { - t.Fatalf("action=%s success=%v", gotAction, gotSuccess) - } - var parsed map[string]interface{} - if err := json.Unmarshal([]byte(gotMsg), &parsed); err != nil { - t.Fatal(err) - } - ports, _ := parsed["open_ports"].([]interface{}) - if len(ports) != 2 { - t.Fatalf("ports=%v", ports) - } -} - -func TestHandleReconRelayScanMissingHost(t *testing.T) { - c := &AgentClient{} - c.commandResultHook = func(action string, success bool, _ string) { - if action != "recon_relay_scan" || success { - t.Fatalf("action=%s success=%v", action, success) - } - } - if !c.handleReconCommand("recon_relay_scan", "", "") { - t.Fatal("expected handled") - } -} -''' - - -def patch_file(rel: str, old: str, new: str) -> None: - path = ROOT / rel - text = path.read_text(encoding="utf-8") - if old not in text: - if new.strip() in text: - return - raise SystemExit(f"patch miss in {rel}: {old[:60]!r}") - path.write_text(text.replace(old, new, 1), encoding="utf-8", newline="\n") - - -def main() -> None: - for rel, content in FILES.items(): - path = ROOT / rel - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8", newline="\n") - print("wrote", rel) - - types_path = ROOT / "server/internal/recon/types.go" - types_text = types_path.read_text(encoding="utf-8") - if "PathTracerHints" not in types_text: - types_text = types_text.replace( - "\tRelayVia string `json:\"relay_via,omitempty\"`\n" - "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", - "\tRelayVia string `json:\"relay_via,omitempty\"`\n" - "\tUDPHints []UDPHint `json:\"udp_hints,omitempty\"`\n" - "\tPathTracerHints []string `json:\"path_tracer_hints,omitempty\"`\n" - "\tMessage string `json:\"message,omitempty\"`\n" - "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", - 1, - ) - types_path.write_text(types_text, encoding="utf-8", newline="\n") - - scan_path = ROOT / "server/internal/recon/scan.go" - if "func NormalizeHost" not in scan_path.read_text(encoding="utf-8"): - patch_file( - "server/internal/recon/scan.go", - "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", - "func NormalizeHost(host string) (string, error) {\n\treturn normalizeOwnedHost(host)\n}\n\n" - "func Scan(req ScanRequest) (*ScanReport, error) {\n\thost, err := normalizeOwnedHost(req.Host)", - ) - - router_path = ROOT / "server/internal/api/router.go" - router_text = router_path.read_text(encoding="utf-8") - if '"/recon/relay-scan"' not in router_text: - patch_file( - "server/internal/api/router.go", - '\t\tr.Post("/recon/scan", reconHandler.Scan)\n', - '\t\tr.Post("/recon/scan", reconHandler.Scan)\n\t\tr.Post("/recon/relay-scan", reconHandler.RelayScan)\n', - ) - - cmd_path = ROOT / "agent/client/commands_common.go" - cmd_text = cmd_path.read_text(encoding="utf-8") - if "recon_relay_scan" not in cmd_text: - patch_file( - "agent/client/commands_common.go", - "func (c *AgentClient) handleReconCommand(action, command string) bool {\n", - "func (c *AgentClient) handleReconCommand(action, command, path string) bool {\n" - "\tif action == \"recon_relay_scan\" {\n" - "\t\thost := strings.TrimSpace(command)\n" - "\t\tif host == \"\" {\n" - "\t\t\tc.sendCommandResult(action, false, \"host is required in command field\")\n" - "\t\t\treturn true\n" - "\t\t}\n" - "\t\tudpGuess := strings.EqualFold(strings.TrimSpace(path), \"true\")\n" - "\t\tout, err := deploy.RunRelayScanJSON(host, udpGuess)\n" - "\t\tif err != nil {\n" - "\t\t\tc.sendCommandResult(action, false, err.Error())\n" - "\t\t\treturn true\n" - "\t\t}\n" - "\t\tc.sendCommandResult(action, true, out)\n" - "\t\treturn true\n" - "\t}\n", - ) - - client_path = ROOT / "agent/client/client.go" - if "handleReconCommand(action, command, path)" not in client_path.read_text(encoding="utf-8"): - patch_file( - "agent/client/client.go", - "\t\tif c.handleReconCommand(action, command) {\n", - "\t\tif c.handleReconCommand(action, command, path) {\n", - ) - - natpunch = (ROOT / "agent/deploy/natpunch.go").read_text(encoding="utf-8") - if "SetProbePortsHook" not in natpunch: - patch_file( - "agent/deploy/natpunch.go", - "var probePortsFn func(host string, ports []int) []int\n\nfunc probePorts", - "var probePortsFn func(host string, ports []int) []int\n\n" - "func SetProbePortsHook(fn func(host string, ports []int) []int) { probePortsFn = fn }\n\n" - "func probePorts", - ) - - print("patches applied") - - -def run(cmd: list[str], cwd: Path | None = None) -> None: - print("+", " ".join(cmd)) - subprocess.run(cmd, cwd=cwd or ROOT, check=True) - - -def main_and_ship() -> None: - main() - - paths = [ - "server/internal/recon/relay_scan.go", - "server/internal/recon/relay_scan_test.go", - "server/internal/recon/types.go", - "server/internal/recon/scan.go", - "server/internal/api/recon_relay_scan.go", - "server/internal/api/recon_relay_scan_test.go", - "server/internal/api/router.go", - "agent/deploy/relay_scan.go", - "agent/deploy/natpunch.go", - "agent/client/commands_common.go", - "agent/client/client.go", - "agent/client/recon_relay_scan_test.go", - "server/web/src/types/recon.ts", - ] - for rel in paths: - if not (ROOT / rel).exists(): - raise SystemExit(f"missing before commit: {rel}") - - run(["git", "add", *paths]) - msg = ( - "Recon network batch 2: fleet relay scan and UDP hints.\n\n" - "POST /api/v1/recon/relay-scan probes from the dashboard when reachable, " - "otherwise dispatches recon_relay_scan to a same-/24 online agent. " - "Agents reuse probePortsFn for TCP and optionally UDP 53/51820 with " - "dns/wireguard Path Tracer tags." - ) - run(["git", "commit", "-m", msg]) - run(["git", "push", "origin", "HEAD"]) - out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() - print("COMMIT_HASH=" + out) - - -if __name__ == "__main__": - # Include agent relay_scan.go in FILES if missing above - FILES.setdefault( - "agent/deploy/relay_scan.go", - (ROOT / "agent/deploy/relay_scan.go").read_text(encoding="utf-8") - if (ROOT / "agent/deploy/relay_scan.go").exists() - else r'''package deploy - -import ( - "encoding/json" - "net" - "strconv" - "strings" - "time" -) - -var RelayScanUDPPorts = []int{53, 51820} - -type RelayScanResult struct { - Host string `json:"host"` - OpenPorts []int `json:"open_ports"` - UDPHints []RelayUDPHint `json:"udp_hints,omitempty"` -} - -type RelayUDPHint struct { - Port int `json:"port"` - Open bool `json:"open"` - Service string `json:"service"` -} - -var udpGuessPortsFn func(host string, ports []int) []RelayUDPHint - -func SetRelayUDPGuessHook(fn func(host string, ports []int) []RelayUDPHint) { udpGuessPortsFn = fn } - -func RunRelayScan(host string, udpGuess bool) RelayScanResult { - host = strings.TrimSpace(host) - open := probePorts(host, SubnetReconPorts) - result := RelayScanResult{Host: host, OpenPorts: append([]int(nil), open...)} - if udpGuess { - result.UDPHints = guessRelayUDPPorts(host, RelayScanUDPPorts) - } - return result -} - -func RunRelayScanJSON(host string, udpGuess bool) (string, error) { - b, err := json.Marshal(RunRelayScan(host, udpGuess)) - if err != nil { - return "", err - } - return string(b), nil -} - -func guessRelayUDPPorts(host string, ports []int) []RelayUDPHint { - if udpGuessPortsFn != nil { - return udpGuessPortsFn(host, ports) - } - var out []RelayUDPHint - for _, port := range ports { - out = append(out, RelayUDPHint{Port: port, Open: probeRelayUDPQuick(host, port), Service: relayUDPServiceLabel(port)}) - } - return out -} - -func relayUDPServiceLabel(port int) string { - switch port { - case 53: - return "dns" - case 51820: - return "wireguard" - default: - return "" - } -} - -func probeRelayUDPQuick(host string, port int) bool { - conn, err := net.DialTimeout("udp", net.JoinHostPort(host, strconv.Itoa(port)), 800*time.Millisecond) - if err != nil { - return false - } - defer conn.Close() - _ = conn.SetDeadline(time.Now().Add(800 * time.Millisecond)) - if b := relayUDPProbePayload(port); len(b) > 0 { - _, _ = conn.Write(b) - } - buf := make([]byte, 512) - n, err := conn.Read(buf) - return err == nil && n > 0 -} - -func relayUDPProbePayload(port int) []byte { - switch port { - case 53: - return []byte{0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x07, 'v', 'e', 'r', 's', 'i', 'o', 'n', 0x04, 'b', 'i', 'n', 'd', 0x00, 0x00, 0x10, 0x00, 0x03} - case 51820: - return []byte{0x01, 0x00, 0x00, 0x00} - default: - return nil - } -} -''', - ) - - recon_ts = ROOT / "server/web/src/types/recon.ts" - text = recon_ts.read_text(encoding="utf-8") - if "relay_via?:" not in text: - text = text.replace( - " crawl?: ReconCrawlReport;\n recommendations?: ReconDeployRecommendation[];\n}", - " crawl?: ReconCrawlReport;\n relay_via?: string;\n" - " udp_hints?: { port: number; open: boolean; service?: string }[];\n" - " path_tracer_hints?: string[];\n message?: string;\n" - " recommendations?: ReconDeployRecommendation[];\n}", - 1, - ) - recon_ts.write_text(text, encoding="utf-8", newline="\n") - - try: - main_and_ship() - except subprocess.CalledProcessError as exc: - sys.exit(exc.returncode) diff --git a/scripts/commit-recon-crawl-batch2.py b/scripts/commit-recon-crawl-batch2.py deleted file mode 100644 index d0ccfe4..0000000 --- a/scripts/commit-recon-crawl-batch2.py +++ /dev/null @@ -1,685 +0,0 @@ -#!/usr/bin/env python3 -"""Write recon crawl batch 2 (upload hunter + admin surface), test, commit, push.""" -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - -ADMIN_SURFACE = r'''package recon - -import ( - "fmt" - "net/http" - "sort" - "strings" -) - -var adminSurfacePaths = []string{ - "/wp-admin", - "/wp-admin/", - "/admin", - "/admin/", - "/admin/login", - "/administrator", - "/api", - "/api/", - "/api/v1", - "/graphql", - "/graphql/", - "/swagger", - "/swagger/", - "/swagger/index.html", - "/swagger-ui", - "/swagger-ui/", - "/actuator", - "/actuator/", - "/actuator/health", - "/.env", - "/.env.local", - "/server-status", - "/server-status/", -} - -func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFinding { - scheme = normalizeScheme(scheme, port) - if port <= 0 { - port = defaultPortForScheme(scheme) - } - base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) - - seen := map[string]bool{} - var out []AdminSurfaceFinding - for _, path := range adminSurfacePaths { - key := strings.ToLower(path) - if seen[key] { - continue - } - seen[key] = true - - rawURL := strings.TrimRight(base, "/") + path - status, _, err := fetchPage(rawURL) - if err != nil { - continue - } - signal := adminSurfaceSignal(status) - if signal == "" { - continue - } - out = append(out, AdminSurfaceFinding{ - Path: path, - URL: rawURL, - StatusCode: status, - Signal: signal, - }) - } - - sort.Slice(out, func(i, j int) bool { - if out[i].Signal != out[j].Signal { - return out[i].Signal == "green" - } - return out[i].Path < out[j].Path - }) - return out -} - -func adminSurfaceSignal(status int) string { - switch status { - case http.StatusOK: - return "green" - case http.StatusUnauthorized, http.StatusForbidden: - return "gray" - default: - return "" - } -} -''' - -UPLOAD_HUNTER = r'''package recon - -import ( - "net/url" - "regexp" - "sort" - "strings" -) - -var ( - openUploadPathRe = regexp.MustCompile(`(?i)(/api/[^\s"'` + "`" + r`<>]*upload|/upload[^\s"'` + "`" + r`<>]*|/v\d+/upload)`) - dragDropClassRe = regexp.MustCompile(`(?i)(dropzone|drop-zone|file-drop|drag-drop|fileupload)`) - jsUploadHintRe = regexp.MustCompile(`(?i)(multipart/form-data|formdata\s*\(|type\s*:\s*['"]file['"]|/api/[^\s"'` + "`" + r`<>]*upload|\.upload\s*\(|dropzone)`) -) - -func collectUploadFromPage(pageURL string, fileInputs, multipart []FormFinding) []UploadHunterFinding { - var out []UploadHunterFinding - for _, f := range fileInputs { - if !f.HasFile { - continue - } - out = append(out, UploadHunterFinding{ - PageURL: pageURL, - Target: resolveUploadTarget(pageURL, f.Action), - Source: "file_input", - Method: f.Method, - }) - } - for _, f := range multipart { - if !f.Multipart { - continue - } - out = append(out, UploadHunterFinding{ - PageURL: pageURL, - Target: resolveUploadTarget(pageURL, f.Action), - Source: "multipart", - Method: f.Method, - }) - } - return out -} - -func detectDragDropZones(pageURL, body string) []UploadHunterFinding { - root, err := htmlParseRoot(body) - if err != nil { - return detectDragDropFromText(pageURL, body) - } - var out []UploadHunterFinding - var walk func(*htmlNode) - walk = func(n *htmlNode) { - if n.tag != "" { - cls := strings.ToLower(n.attr("class")) - id := strings.ToLower(n.attr("id")) - dropAttr := strings.ToLower(n.attr("data-dropzone")) - if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || dropAttr != "" { - target := n.attr("data-upload-url") - if target == "" { - target = n.attr("action") - } - out = append(out, UploadHunterFinding{ - PageURL: pageURL, - Target: resolveUploadTarget(pageURL, target), - Source: "drag_drop", - }) - } - } - for _, c := range n.children { - walk(c) - } - } - walk(root) - if len(out) == 0 { - return detectDragDropFromText(pageURL, body) - } - return out -} - -func detectDragDropFromText(pageURL, body string) []UploadHunterFinding { - lower := strings.ToLower(body) - if !strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag") { - return nil - } - if !strings.Contains(lower, "upload") && !strings.Contains(lower, "file") { - return nil - } - return []UploadHunterFinding{{ - PageURL: pageURL, - Source: "drag_drop", - }} -} - -func extractScriptSrc(pageURL, body string) []string { - root, err := htmlParseRoot(body) - if err != nil { - return nil - } - var srcs []string - var walk func(*htmlNode) - walk = func(n *htmlNode) { - if n.tag == "script" { - if src := strings.TrimSpace(n.attr("src")); src != "" { - srcs = append(srcs, src) - } - } - for _, c := range n.children { - walk(c) - } - } - walk(root) - _ = pageURL - return srcs -} - -func scanJSForUpload(jsURL, body string) []UploadHunterFinding { - if !jsUploadHintRe.MatchString(body) { - return nil - } - target := jsURL - if m := openUploadPathRe.FindString(body); m != "" { - target = m - } - return []UploadHunterFinding{{ - PageURL: jsURL, - Target: target, - Source: "js", - }} -} - -func isScriptAsset(ref string) bool { - ref = strings.ToLower(strings.TrimSpace(ref)) - return strings.HasSuffix(ref, ".js") || strings.Contains(ref, ".js?") -} - -func resolveUploadTarget(pageURL, action string) string { - action = strings.TrimSpace(action) - if action == "" { - if u, err := url.Parse(pageURL); err == nil { - return u.Path - } - return pageURL - } - if strings.HasPrefix(action, "http://") || strings.HasPrefix(action, "https://") { - return action - } - if abs, err := resolveSameOrigin(pageURL, action); err == nil { - return abs - } - return action -} - -func rankUploadFindings(base string, in []UploadHunterFinding) []UploadHunterFinding { - if len(in) == 0 { - return nil - } - seen := map[string]UploadHunterFinding{} - for _, f := range in { - key := strings.ToLower(f.PageURL + "|" + f.Target + "|" + f.Source) - if prev, ok := seen[key]; ok { - if scoreUploadFinding(f) > scoreUploadFinding(prev) { - seen[key] = f - } - continue - } - seen[key] = f - } - out := make([]UploadHunterFinding, 0, len(seen)) - for _, f := range seen { - f = tagUploadFinding(base, f) - f.Score = scoreUploadFinding(f) - out = append(out, f) - } - sort.Slice(out, func(i, j int) bool { - if out[i].Score != out[j].Score { - return out[i].Score > out[j].Score - } - return out[i].Target < out[j].Target - }) - return out -} - -func tagUploadFinding(base string, f UploadHunterFinding) UploadHunterFinding { - target := f.Target - if target == "" { - target = f.PageURL - } - path := target - if u, err := url.Parse(target); err == nil && u.Path != "" { - path = u.Path - } - if openUploadPathRe.MatchString(path) || openUploadPathRe.MatchString(target) { - f.Tags = appendUniqueTag(f.Tags, "open_api") - } - probeURL := target - if !strings.HasPrefix(probeURL, "http://") && !strings.HasPrefix(probeURL, "https://") { - if abs, err := resolveSameOrigin(base, probeURL); err == nil { - probeURL = abs - } else { - probeURL = strings.TrimRight(base, "/") + "/" + strings.TrimLeft(probeURL, "/") - } - } - status, _, err := fetchPage(probeURL) - if err == nil { - f.StatusCode = status - if status == 200 { - f.Tags = appendUniqueTag(f.Tags, "no_auth") - } - } - return f -} - -func scoreUploadFinding(f UploadHunterFinding) int { - score := 10 - switch f.Source { - case "multipart": - score += 30 - case "file_input": - score += 25 - case "drag_drop": - score += 20 - case "js": - score += 15 - } - for _, tag := range f.Tags { - switch tag { - case "no_auth": - score += 40 - case "open_api": - score += 25 - } - } - if f.StatusCode == 200 { - score += 10 - } - return score -} - -func appendUniqueTag(tags []string, tag string) []string { - for _, t := range tags { - if t == tag { - return tags - } - } - return append(tags, tag) -} -''' - -TEST_APPEND = r''' - -func TestUploadHunterMultipartAndJS(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/": - w.Write([]byte(` + "`" + r` -
- -
-
- -` + "`" + r`)) - case "/static/upload.js": - w.Header().Set("Content-Type", "application/javascript") - w.Write([]byte(` + "`" + r`fetch("/api/upload",{method:"POST",body:new FormData()})` + "`" + r`)) - case "/api/upload": - w.WriteHeader(http.StatusOK) - default: - http.NotFound(w, r) - } - })) - defer srv.Close() - - u, err := url.Parse(srv.URL) - if err != nil { - t.Fatal(err) - } - port := 80 - if p := u.Port(); p != "" { - port = atoi(p) - } - - SetFetchPageHook(func(rawURL string) (int, string, error) { - resp, err := http.Get(rawURL) - if err != nil { - return 0, "", err - } - defer resp.Body.Close() - body, _ := readBodyLimited(resp.Body, maxHTMLBytes) - return resp.StatusCode, body, nil - }) - t.Cleanup(func() { SetFetchPageHook(nil) }) - - report, err := Crawl(u.Hostname(), port, u.Scheme, []string{"/"}) - if err != nil { - t.Fatal(err) - } - if len(report.UploadHunter) == 0 { - t.Fatal("expected upload hunter findings") - } - if report.UploadHunter[0].Score <= 0 { - t.Fatalf("score=%d", report.UploadHunter[0].Score) - } - hasOpenAPI := false - hasNoAuth := false - for _, f := range report.UploadHunter { - for _, tag := range f.Tags { - if tag == "open_api" { - hasOpenAPI = true - } - if tag == "no_auth" { - hasNoAuth = true - } - } - } - if !hasOpenAPI { - t.Fatalf("missing open_api tag: %+v", report.UploadHunter) - } - if !hasNoAuth { - t.Fatalf("missing no_auth tag: %+v", report.UploadHunter) - } -} - -func TestProbeAdminSurfaceSignals(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/admin", "/admin/": - w.WriteHeader(http.StatusForbidden) - case "/graphql", "/graphql/": - w.WriteHeader(http.StatusUnauthorized) - case "/swagger", "/swagger/": - w.WriteHeader(http.StatusOK) - w.Write([]byte("swagger-ui")) - default: - http.NotFound(w, r) - } - })) - defer srv.Close() - - u, err := url.Parse(srv.URL) - if err != nil { - t.Fatal(err) - } - port := 80 - if p := u.Port(); p != "" { - port = atoi(p) - } - - SetFetchPageHook(func(rawURL string) (int, string, error) { - resp, err := http.Get(rawURL) - if err != nil { - return 0, "", err - } - defer resp.Body.Close() - body, _ := readBodyLimited(resp.Body, maxHTMLBytes) - return resp.StatusCode, body, nil - }) - t.Cleanup(func() { SetFetchPageHook(nil) }) - - findings := ProbeAdminSurface(u.Hostname(), port, u.Scheme) - byPath := map[string]string{} - for _, f := range findings { - byPath[f.Path] = f.Signal - } - if byPath["/admin"] != "gray" && byPath["/admin/"] != "gray" { - t.Fatalf("admin signal=%v", byPath) - } - if byPath["/graphql"] != "gray" && byPath["/graphql/"] != "gray" { - t.Fatalf("graphql signal=%v", byPath) - } - if byPath["/swagger"] != "green" && byPath["/swagger/"] != "green" { - t.Fatalf("swagger signal=%v", byPath) - } -} - -func TestScanReportIncludesAdminSurfaceJSON(t *testing.T) { - SetPortDialHook(func(host string, port int, _ time.Duration) bool { - return port == 80 - }) - t.Cleanup(func() { SetPortDialHook(nil) }) - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/": - w.Write([]byte(` + "`" + r`
` + "`" + r`)) - case "/api/upload": - w.WriteHeader(http.StatusOK) - case "/admin", "/admin/": - w.WriteHeader(http.StatusForbidden) - default: - http.NotFound(w, r) - } - })) - defer srv.Close() - - u, _ := url.Parse(srv.URL) - port := 80 - if p := u.Port(); p != "" { - port = atoi(p) - } - - SetFetchPageHook(func(rawURL string) (int, string, error) { - resp, err := http.Get(rawURL) - if err != nil { - return 0, "", err - } - defer resp.Body.Close() - body, _ := readBodyLimited(resp.Body, maxHTMLBytes) - return resp.StatusCode, body, nil - }) - t.Cleanup(func() { SetFetchPageHook(nil) }) - - report, err := Scan(ScanRequest{Host: u.Hostname(), Port: port, Scheme: u.Scheme}) - if err != nil { - t.Fatal(err) - } - raw, err := json.Marshal(report) - if err != nil { - t.Fatal(err) - } - payload := string(raw) - if !strings.Contains(payload, `"admin_surface"`) { - t.Fatalf("missing admin_surface: %s", payload) - } - if !strings.Contains(payload, `"upload_hunter"`) { - t.Fatalf("missing upload_hunter: %s", payload) - } - if len(report.AdminSurface) == 0 { - t.Fatal("expected admin surface findings") - } - if report.Crawl == nil || len(report.Crawl.UploadHunter) == 0 { - t.Fatal("expected crawl upload hunter findings") - } -} -''' - - -def patch(path: Path, old: str, new: str) -> None: - text = path.read_text(encoding="utf-8") - if old not in text: - if new.strip() in text: - return - raise SystemExit(f"patch miss in {path}: {old[:80]!r}") - path.write_text(text.replace(old, new, 1), encoding="utf-8", newline="\n") - - -def main() -> None: - recon = ROOT / "server/internal/recon" - (recon / "admin_surface.go").write_text(ADMIN_SURFACE, encoding="utf-8", newline="\n") - (recon / "upload_hunter.go").write_text(UPLOAD_HUNTER, encoding="utf-8", newline="\n") - - types = recon / "types.go" - patch( - types, - "\tCMSFingerprints []string `json:\"cms_fingerprints,omitempty\"`\n}", - "\tCMSFingerprints []string `json:\"cms_fingerprints,omitempty\"`\n" - "\tUploadHunter []UploadHunterFinding `json:\"upload_hunter,omitempty\"`\n}", - ) - patch( - types, - "type PageFinding struct {", - "type UploadHunterFinding struct {\n" - "\tPageURL string `json:\"page_url\"`\n" - "\tTarget string `json:\"target,omitempty\"`\n" - "\tSource string `json:\"source\"`\n" - "\tMethod string `json:\"method,omitempty\"`\n" - "\tTags []string `json:\"tags,omitempty\"`\n" - "\tScore int `json:\"score\"`\n" - "\tStatusCode int `json:\"status_code,omitempty\"`\n" - "}\n\n" - "type AdminSurfaceFinding struct {\n" - "\tPath string `json:\"path\"`\n" - "\tURL string `json:\"url\"`\n" - "\tStatusCode int `json:\"status_code\"`\n" - "\tSignal string `json:\"signal\"`\n" - "}\n\n" - "type PageFinding struct {", - ) - patch( - types, - "\tCrawl *CrawlReport `json:\"crawl,omitempty\"`\n" - "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", - "\tCrawl *CrawlReport `json:\"crawl,omitempty\"`\n" - "\tAdminSurface []AdminSurfaceFinding `json:\"admin_surface,omitempty\"`\n" - "\tRecommendations []DeployRecommendation `json:\"recommendations,omitempty\"`\n}", - ) - - scan = recon / "scan.go" - patch( - scan, - "\t\tif err == nil && crawl != nil {\n" - "\t\t\treport.Crawl = crawl\n" - "\t\t}\n" - "\t}", - "\t\tif err == nil && crawl != nil {\n" - "\t\t\treport.Crawl = crawl\n" - "\t\t}\n" - "\t\treport.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)\n" - "\t}", - ) - patch( - scan, - "\t\tif len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 {", - "\t\tif len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 || len(crawl.UploadHunter) > 0 {", - ) - - crawl = recon / "crawl.go" - patch( - crawl, - "\treport := &CrawlReport{}\n\tvisited := map[string]bool{}", - "\treport := &CrawlReport{}\n\tvar uploadRaw []UploadHunterFinding\n\tvar jsQueue []string\n\tjsSeen := map[string]bool{}\n\tvisited := map[string]bool{}", - ) - patch( - crawl, - "\t\treport.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)\n\n\t\tif item.depth >= DefaultCrawlDepth {", - "\t\treport.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)\n" - "\t\tuploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)\n" - "\t\tuploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)\n\n" - "\t\tif item.depth <= DefaultCrawlDepth {\n" - "\t\t\tfor _, src := range extractScriptSrc(item.url, body) {\n" - "\t\t\t\tabs, err := resolveSameOrigin(base, src)\n" - "\t\t\t\tif err != nil || !sameOrigin(base, abs) || !isScriptAsset(src) {\n" - "\t\t\t\t\tcontinue\n" - "\t\t\t\t}\n" - "\t\t\t\tlkey := normalizeURLKey(abs)\n" - "\t\t\t\tif jsSeen[lkey] {\n" - "\t\t\t\t\tcontinue\n" - "\t\t\t\t}\n" - "\t\t\t\tjsSeen[lkey] = true\n" - "\t\t\t\tjsQueue = append(jsQueue, abs)\n" - "\t\t\t}\n" - "\t\t}\n\n\t\tif item.depth >= DefaultCrawlDepth {", - ) - patch( - crawl, - "\treport.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\treturn report, nil", - "\treport.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)\n\n" - "\tmaxJS := 20\n" - "\tif len(jsQueue) > maxJS {\n" - "\t\tjsQueue = jsQueue[:maxJS]\n" - "\t}\n" - "\tfor _, jsURL := range jsQueue {\n" - "\t\t_, jsBody, err := fetchPage(jsURL)\n" - "\t\tif err != nil {\n" - "\t\t\tcontinue\n" - "\t\t}\n" - "\t\tuploadRaw = append(uploadRaw, scanJSForUpload(jsURL, jsBody)...)\n" - "\t}\n" - "\treport.UploadHunter = rankUploadFindings(base, uploadRaw)\n\treturn report, nil", - ) - - test = recon / "recon_test.go" - text = test.read_text(encoding="utf-8") - if "TestUploadHunterMultipartAndJS" not in text: - marker = "func containsStr(list []string, want string) bool {" - if marker not in text: - raise SystemExit("recon_test.go marker missing") - text = text.replace(marker, TEST_APPEND + "\n" + marker, 1) - test.write_text(text, encoding="utf-8", newline="\n") - - print("files written") - - -def run(cmd: list[str], cwd: Path | None = None) -> None: - print("+", " ".join(cmd)) - subprocess.run(cmd, cwd=cwd or ROOT, check=True) - - -if __name__ == "__main__": - subprocess.run(["git", "checkout", "HEAD", "--", "server/internal/recon/"], cwd=ROOT, check=True) - main() - run(["go", "test", "./internal/recon/...", "-count=1"], cwd=ROOT / "server") - paths = [ - "server/internal/recon/admin_surface.go", - "server/internal/recon/upload_hunter.go", - "server/internal/recon/types.go", - "server/internal/recon/scan.go", - "server/internal/recon/crawl.go", - "server/internal/recon/recon_test.go", - ] - run(["git", "add", *paths]) - msg = ( - "Add recon upload hunter and admin surface probing for owned-target scans.\n\n" - "Extend web crawl with multipart/drag-drop/JS upload ranking and probe common " - "admin paths for 200 vs 401/403 signals in scan JSON." - ) - run(["git", "commit", "-m", msg]) - run(["git", "pull", "--rebase", "origin", "main"]) - run(["git", "push", "origin", "HEAD"]) - out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() - print("COMMIT_HASH=" + out) diff --git a/scripts/commit-recon-ux.py b/scripts/commit-recon-ux.py deleted file mode 100644 index eaf4ca0..0000000 --- a/scripts/commit-recon-ux.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""Recon UX backend: write files, test, commit, push.""" -from pathlib import Path -import subprocess, sys - -ROOT = Path(__file__).resolve().parents[1] -S = ROOT / "server" - -def w(rel, text): - p = S / rel - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(text, encoding="utf-8", newline="\n") - print("wrote", rel) - -w("internal/recon/types.go", Path(__file__).with_name("types.go").read_text(encoding="utf-8")) -w("internal/recon/recon_ux.go", Path(__file__).with_name("recon_ux.go").read_text(encoding="utf-8")) -w("internal/recon/scan.go", Path(__file__).with_name("scan.go").read_text(encoding="utf-8")) -w("internal/db/recon_scans.go", Path(__file__).with_name("recon_scans.go").read_text(encoding="utf-8")) -w("internal/api/recon_handler.go", Path(__file__).with_name("recon_handler.go").read_text(encoding="utf-8")) -w("internal/api/recon_handler_test.go", Path(__file__).with_name("recon_handler_test.go").read_text(encoding="utf-8")) -w("internal/recon/recon_stream_test.go", Path(__file__).with_name("recon_stream_test.go").read_text(encoding="utf-8")) - -# crawl delegate -crawl = (S / "internal/recon/crawl.go").read_text(encoding="utf-8") -if "crawlWithOptions" not in crawl.split("func Crawl(")[1].split("\nfunc ")[0]: - crawl = crawl.replace( - "func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\tscheme = normalizeScheme", - "func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\treturn crawlWithOptions(host, port, scheme, seedPaths, ProfileOptions(\"\"), \"\", nil)\n}\n\nfunc crawlLegacy(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {\n\tscheme = normalizeScheme", - 1, - ) - w("internal/recon/crawl.go", crawl) - -sqlite = (S / "internal/db/sqlite.go").read_text(encoding="utf-8") -if "ensureReconScansTable" not in sqlite: - sqlite = sqlite.replace( - "\tif err := d.ensureSeerTables(); err != nil {\n\t\treturn fmt.Errorf(\"seer migration: %w\", err)\n\t}\n", - "\tif err := d.ensureSeerTables(); err != nil {\n\t\treturn fmt.Errorf(\"seer migration: %w\", err)\n\t}\n\tif err := d.ensureReconScansTable(); err != nil {\n\t\treturn fmt.Errorf(\"recon_scans migration: %w\", err)\n\t}\n", - 1, - ) - w("internal/db/sqlite.go", sqlite) - -router = (S / "internal/api/router.go").read_text(encoding="utf-8") -if "/recon/history" not in router: - router = router.replace( - "\t\tr.Post(\"/recon/scan\", reconHandler.Scan)\n", - "\t\tr.Post(\"/recon/scan\", reconHandler.Scan)\n\t\tr.Get(\"/recon/history\", reconHandler.History)\n\t\tr.Get(\"/recon/export/{scan_id}\", reconHandler.Export)\n", - 1, - ) - w("internal/api/router.go", router) - -# fix NewReconHandler call if 3-arg -router = (S / "internal/api/router.go").read_text(encoding="utf-8") -if "NewReconHandler(database, wsHub, publicURLOverride)" in router: - router = router.replace("NewReconHandler(database, wsHub, publicURLOverride)", "NewReconHandler(database, wsHub)") - w("internal/api/router.go", router) - -# websocket test append -wt = S / "internal/api/websocket_test.go" -body = wt.read_text(encoding="utf-8") -if "TestWSReconStreamingEventTypes" not in body: - if '"bytes"' not in body.split("import (")[1].split(")")[0]: - body = body.replace('"encoding/base64"', '"bytes"\n\t"encoding/base64"', 1) - if '"crypto-miner-server/internal/recon"' not in body: - body = body.replace('"crypto-miner-server/internal/pool"', '"crypto-miner-server/internal/pool"\n\t"crypto-miner-server/internal/recon"', 1) - body = body.rstrip() + "\n\n" + Path(__file__).with_name("websocket_test_snippet.go").read_text(encoding="utf-8").split("package api\n", 1)[1] - w("internal/api/websocket_test.go", body) - -env = {**dict(os.environ), "GOCACHE": str(ROOT / ".gocache")} if False else None -r = subprocess.run( - ["go", "test", "./internal/recon/...", "./internal/api/...", "-count=1", "-run", "Recon|WSRecon|ScanStream|DiffReports|ProfileOptions|BuildHistory"], - cwd=S, -) -if r.returncode != 0: - sys.exit(r.returncode) - -subprocess.run(["git", "pull", "origin", "main"], cwd=ROOT, check=False) -subprocess.run(["git", "add", "-A"], cwd=ROOT, check=True) -subprocess.run(["git", "commit", "-m", "Add recon UX backend: streaming scans, profiles, history, export."], cwd=ROOT, check=True) -subprocess.run(["git", "push", "origin", "main"], cwd=ROOT, check=True) -print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()) diff --git a/scripts/install_batch1_atomic.py b/scripts/install_batch1_atomic.py deleted file mode 100644 index 2498440..0000000 --- a/scripts/install_batch1_atomic.py +++ /dev/null @@ -1,404 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import sys -from pathlib import Path - -ROOT = Path(r"G:/crypto miner") -SERVER = ROOT / "server" - - -def wb(rel, text): - p = ROOT / rel - p.parent.mkdir(parents=True, exist_ok=True) - data = text.encode("utf-8") - if b"\x00" in data: - raise SystemExit("NUL in " + rel) - p.write_bytes(data) - - -def patch(rel, old, new): - p = ROOT / rel - t = p.read_text(encoding="utf-8") - if old not in t: - if new in t: - return - raise SystemExit("patch miss " + rel) - p.write_text(t.replace(old, new, 1), encoding="utf-8", newline="\n") - - -types = (ROOT / "scripts/install_batch1_final.py").read_text(encoding="utf-8") -# types block is embedded in install_batch1_final - write from known good merge -types = '''package recon - -import "time" - -var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} - -const ( - DefaultPortDialTimeout = 2 * time.Second - DefaultCrawlDepth = 2 - DefaultCrawlMaxPages = 50 -) - -type ScanRequest struct { - Host string `json:"host"` - Port int `json:"port,omitempty"` - Scheme string `json:"scheme,omitempty"` - Paths []string `json:"paths,omitempty"` - Profile string `json:"profile,omitempty"` - Profiles []string `json:"profiles,omitempty"` - SSRFCanary bool `json:"ssrf_canary,omitempty"` -} - -type PortResult struct { - Port int `json:"port"` - Open bool `json:"open"` -} - -type UDPHint struct { - Port int `json:"port"` - Open bool `json:"open"` - Service string `json:"service"` -} - -type PortBanner struct { - Port int `json:"port"` - Service string `json:"service,omitempty"` - Banner string `json:"banner,omitempty"` - Title string `json:"title,omitempty"` - Hint string `json:"hint,omitempty"` -} - -type StackEntry struct { - Name string `json:"name"` - Source string `json:"source"` - Detail string `json:"detail,omitempty"` -} - -type HTTPHeaderSnap struct { - URL string - Headers map[string]string -} - -type FormFinding struct { - PageURL string `json:"page_url"` - Action string `json:"action,omitempty"` - Method string `json:"method,omitempty"` - Enctype string `json:"enctype,omitempty"` - Fields []string `json:"fields,omitempty"` - HasFile bool `json:"has_file_input,omitempty"` - Multipart bool `json:"multipart,omitempty"` -} - -type URLFieldFinding struct { - PageURL string `json:"page_url"` - Name string `json:"name"` - ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` - Hint string `json:"hint"` -} - -type FormFieldFingerprint struct { - PageURL string `json:"page_url"` - Name string `json:"name"` - ID string `json:"id,omitempty"` - Placeholder string `json:"placeholder,omitempty"` - Score int `json:"score"` - Matches []string `json:"matches,omitempty"` - PasteTarget string `json:"paste_target,omitempty"` -} - -type SSRFCanaryInfo struct { - ScanID string `json:"scan_id"` - URL string `json:"url"` - Status string `json:"status"` - PasteTarget string `json:"paste_target"` - PasteFieldName string `json:"paste_field_name,omitempty"` - PasteFieldID string `json:"paste_field_id,omitempty"` - HitAt *time.Time `json:"hit_at,omitempty"` -} - -type UploadHunterFinding struct { - PageURL string `json:"page_url"` - Target string `json:"target,omitempty"` - Source string `json:"source"` - Method string `json:"method,omitempty"` - Tags []string `json:"tags,omitempty"` - Score int `json:"score"` - StatusCode int `json:"status_code,omitempty"` -} - -type AdminSurfaceFinding struct { - Path string `json:"path"` - URL string `json:"url"` - StatusCode int `json:"status_code"` - Signal string `json:"signal"` -} - -type PageFinding struct { - URL string `json:"url"` - StatusCode int `json:"status_code"` - Title string `json:"title,omitempty"` -} - -type CrawlReport struct { - PagesFetched int `json:"pages_fetched"` - Pages []PageFinding `json:"pages,omitempty"` - FileInputs []FormFinding `json:"file_inputs,omitempty"` - MultipartForms []FormFinding `json:"multipart_forms,omitempty"` - URLFields []URLFieldFinding `json:"url_fields,omitempty"` - FingerprintFields []FormFieldFingerprint `json:"fingerprint_fields,omitempty"` - SSRFScore int `json:"ssrf_score"` - CMSFingerprints []string `json:"cms_fingerprints,omitempty"` - Stack []StackEntry `json:"stack,omitempty"` - UploadHunter []UploadHunterFinding `json:"upload_hunter,omitempty"` -} - -type DeployRecommendation struct { - Lane string `json:"lane,omitempty"` - Template string `json:"template,omitempty"` - Reason string `json:"reason"` - Priority int `json:"priority"` -} - -type ReconScanDiff struct { - NewPorts []int `json:"new_ports,omitempty"` - NewForms []FormFinding `json:"new_forms,omitempty"` -} - -type ReconHistoryEntry struct { - ScanID string `json:"scan_id"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - Status string `json:"status,omitempty"` - ScannedAt time.Time `json:"scanned_at"` - Report *ScanReport `json:"report"` - Diff *ReconScanDiff `json:"diff,omitempty"` -} - -type ScanReport struct { - ScanID string `json:"scan_id,omitempty"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - ProfilesUsed []string `json:"profiles_used,omitempty"` - Status string `json:"status,omitempty"` - ScannedAt time.Time `json:"scanned_at"` - Ports []PortResult `json:"ports"` - Banners []PortBanner `json:"banners,omitempty"` - Stack []StackEntry `json:"stack,omitempty"` - DeployKitLane string `json:"deploy_kit_lane,omitempty"` - Crawl *CrawlReport `json:"crawl,omitempty"` - AdminSurface []AdminSurfaceFinding `json:"admin_surface,omitempty"` - RelayVia string `json:"relay_via,omitempty"` - UDPHints []UDPHint `json:"udp_hints,omitempty"` - PathTracerHints []string `json:"path_tracer_hints,omitempty"` - Message string `json:"message,omitempty"` - Recommendations []DeployRecommendation `json:"recommendations,omitempty"` - Canary *SSRFCanaryInfo `json:"canary,omitempty"` -} -''' - -stub = ROOT / "server/internal/recon/fingerprint_stub.go" -if stub.exists(): - stub.unlink() - -wb("server/internal/recon/types.go", types) -wb("server/internal/recon/fingerprint.go", (ROOT / "scripts/_fingerprint.go.bak").read_text(encoding="utf-8")) -wb("server/internal/recon/canary.go", (ROOT / "server/internal/recon/canary.go").read_text(encoding="utf-8") if (ROOT / "server/internal/recon/canary.go").exists() and "BuildSSRfCanaryURL" in (ROOT / "server/internal/recon/canary.go").read_text(encoding="utf-8") else '''package recon - -import ("fmt"; "net/url"; "strings") - -func BuildSSRfCanaryURL(publicBase, scanID string) string { - base := strings.TrimSpace(publicBase) - base = strings.TrimSuffix(base, "/") - if base == "" { base = "https://localhost" } - if strings.HasPrefix(base, "http://") { base = "https://" + strings.TrimPrefix(base, "http://") } else if !strings.HasPrefix(base, "https://") { base = "https://" + base } - u, err := url.Parse(base) - if err != nil { return fmt.Sprintf("https://localhost/recon/ping/%s", scanID) } - u.Scheme = "https"; u.Path = "/recon/ping/" + scanID; u.RawQuery = ""; u.Fragment = "" - return u.String() -} -''') - -patch("server/internal/recon/parse.go", "Name: label,\n\t\t\t\tType:", "Name: label,\n\t\t\t\tID: id,\n\t\t\t\tType:") -for rel in ("server/internal/recon/crawl.go", "server/internal/recon/scan.go"): - patch(rel, "report.SSRFScore += pageScore\n", "report.SSRFScore += pageScore\n\t\tAppendPageFingerprints(report, item.url, body)\n") - -wb("server/internal/db/recon_canary.go", (ROOT / "server/internal/db/recon_canary.go").read_text(encoding="utf-8") if (ROOT / "server/internal/db/recon_canary.go").exists() and "MarkReconSSRfCanaryHit" in (ROOT / "server/internal/db/recon_canary.go").read_text(encoding="utf-8") else Path(r"G:/crypto miner/server/internal/db/recon_canary.go").read_text(encoding="utf-8")) - -# always write clean db if file missing key func - simpler: always overwrite from backup content in handler write above -# Re-read - use explicit content -wb("server/internal/db/recon_canary.go", '''package db - -import ( - "database/sql" - "time" - - "crypto-miner-server/internal/recon" -) - -func (d *Database) ensureReconSSRfCanaryTable() error { - _, err := d.Exec(`CREATE TABLE IF NOT EXISTS recon_ssrf_canary ( - scan_id TEXT PRIMARY KEY, host TEXT NOT NULL, canary_url TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', paste_field_name TEXT NOT NULL DEFAULT '', - paste_field_id TEXT NOT NULL DEFAULT '', hit_at DATETIME, created_at DATETIME NOT NULL)`) - return err -} - -func (d *Database) InsertReconSSRfCanary(scanID, host, canaryURL, pasteName, pasteID string) error { - if err := d.ensureReconSSRfCanaryTable(); err != nil { return err } - _, err := d.Exec(`INSERT INTO recon_ssrf_canary (scan_id, host, canary_url, status, paste_field_name, paste_field_id, created_at) VALUES (?, ?, ?, 'pending', ?, ?, ?)`, - scanID, host, canaryURL, pasteName, pasteID, time.Now().UTC().Format(time.RFC3339Nano)) - return err -} - -func (d *Database) GetReconSSRfCanary(scanID string) (*recon.SSRFCanaryInfo, error) { - if err := d.ensureReconSSRfCanaryTable(); err != nil { return nil, err } - var host, canaryURL, status, pasteName, pasteID string - var hitAt sql.NullString - err := d.QueryRow(`SELECT host, canary_url, status, paste_field_name, paste_field_id, hit_at FROM recon_ssrf_canary WHERE scan_id = ?`, scanID).Scan(&host, &canaryURL, &status, &pasteName, &pasteID, &hitAt) - if err != nil { return nil, err } - info := &recon.SSRFCanaryInfo{ScanID: scanID, URL: canaryURL, Status: status, PasteTarget: canaryURL, PasteFieldName: pasteName, PasteFieldID: pasteID} - if hitAt.Valid { if ts, err := time.Parse(time.RFC3339Nano, hitAt.String); err == nil { info.HitAt = &ts } } - return info, nil -} - -func (d *Database) UpdateReconSSRfCanaryPasteField(scanID, pasteName, pasteID string) error { - if err := d.ensureReconSSRfCanaryTable(); err != nil { return err } - _, err := d.Exec(`UPDATE recon_ssrf_canary SET paste_field_name = ?, paste_field_id = ? WHERE scan_id = ?`, pasteName, pasteID, scanID) - return err -} - -func (d *Database) MarkReconSSRfCanaryHit(scanID string) (bool, error) { - if err := d.ensureReconSSRfCanaryTable(); err != nil { return false, err } - res, err := d.Exec(`UPDATE recon_ssrf_canary SET status = 'confirmed', hit_at = ? WHERE scan_id = ? AND status != 'confirmed'`, time.Now().UTC().Format(time.RFC3339Nano), scanID) - if err != nil { return false, err } - if n, _ := res.RowsAffected(); n > 0 { return true, nil } - var status string - err = d.QueryRow(`SELECT status FROM recon_ssrf_canary WHERE scan_id = ?`, scanID).Scan(&status) - return status == "confirmed", err -} -''') - -wb("server/internal/db/recon_canary_test.go", '''package db - -import "testing" - -func TestReconSSRfCanaryInsertAndHit(t *testing.T) { - d, err := New(t.TempDir()) - if err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = d.Close() }) - if err := d.InsertReconSSRfCanary("c1", "lab", "https://x/recon/ping/c1", "u", "i"); err != nil { t.Fatal(err) } - info, err := d.GetReconSSRfCanary("c1") - if err != nil || info.Status != "pending" { t.Fatal(info, err) } - hit, err := d.MarkReconSSRfCanaryHit("c1") - if err != nil || !hit { t.Fatal(hit, err) } - info, _ = d.GetReconSSRfCanary("c1") - if info.Status != "confirmed" || info.HitAt == nil { t.Fatal(info) } -} -''') - -wb("server/internal/api/recon_canary_hub.go", '''package api - -import ("sync"; "time"; "crypto-miner-server/internal/recon") - -type ReconCanaryHub struct { mu sync.RWMutex; byID map[string]*recon.SSRFCanaryInfo } - -func NewReconCanaryHub() *ReconCanaryHub { return &ReconCanaryHub{byID: map[string]*recon.SSRFCanaryInfo{}} } - -func (h *ReconCanaryHub) Register(id string, info *recon.SSRFCanaryInfo) { - if h == nil || info == nil { return } - h.mu.Lock(); defer h.mu.Unlock(); cp := *info; h.byID[id] = &cp -} - -func (h *ReconCanaryHub) Get(id string) *recon.SSRFCanaryInfo { - if h == nil { return nil } - h.mu.RLock(); defer h.mu.RUnlock() - if x := h.byID[id]; x != nil { cp := *x; return &cp } - return nil -} - -func (h *ReconCanaryHub) MarkHit(id string) bool { - if h == nil { return false } - h.mu.Lock(); defer h.mu.Unlock() - x, ok := h.byID[id]; if !ok { return false } - if x.Status == "confirmed" { return true } - now := time.Now().UTC(); x.Status = "confirmed"; x.HitAt = &now; return true -} - -func (h *ReconCanaryHub) UpdatePasteField(id, name, id2 string) { - if h == nil { return } - h.mu.Lock(); defer h.mu.Unlock() - if x := h.byID[id]; x != nil { x.PasteFieldName, x.PasteFieldID = name, id2 } -} - -func BroadcastReconCanaryHit(hub *WSHub, scanID string) { - if hub == nil { return } - hub.broadcastDashboard(Message{Type: "recon_canary_hit", Payload: mustMarshal(map[string]interface{}{"scan_id": scanID, "status": "confirmed"})}) -} -''') - -wb("server/internal/api/recon_handler.go", (ROOT / "scripts/_recon_handler_canary.go.bak").read_text(encoding="utf-8")) -ht = (ROOT / "scripts/_recon_handler_test_canary.go.bak").read_text(encoding="utf-8") -if '"time"' not in ht: - ht = ht.replace('"testing"\n', '"testing"\n\t"time"\n') -wb("server/internal/api/recon_handler_test.go", ht) - -rtest = (ROOT / "server/internal/recon/recon_test.go").read_text(encoding="utf-8") -if "TestBuildSSRfCanaryURLForcesHTTPS" not in rtest: - rtest += ''' - -func TestScoreFormFieldFingerprintKeywords(t *testing.T) { - score, matches := ScoreFormFieldFingerprint("callback_url", "", "") - if score < 14 || !containsStr(matches, "callback") { t.Fatalf("score=%d matches=%v", score, matches) } -} - -func TestExtractFormFieldFingerprintsExactNameID(t *testing.T) { - fps := ExtractFormFieldFingerprints("http://lab/", ``) - if len(fps) == 0 || fps[0].Name != "avatar_url" || fps[0].ID != "userAvatar" { t.Fatalf("%+v", fps) } -} - -func TestBuildSSRfCanaryURLForcesHTTPS(t *testing.T) { - u := BuildSSRfCanaryURL("http://public.example.com", "abc-123") - if !strings.HasPrefix(u, "https://") || !strings.Contains(u, "/recon/ping/abc-123") { t.Fatalf("url=%q", u) } -} -''' -wb("server/internal/recon/recon_test.go", rtest) - -rt = (ROOT / "server/internal/api/router.go").read_text(encoding="utf-8") -if "recon/canary" not in rt: - rt = rt.replace('r.Post("/recon/scan", reconHandler.Scan)', 'r.Post("/recon/scan", reconHandler.Scan)\n\t\tr.Get("/recon/canary/{scan_id}", reconHandler.CanaryStatus)', 1) -if "NewReconHandler(database, wsHub, publicURLOverride)" not in rt: - rt = rt.replace("NewReconHandler(database, wsHub)", "NewReconHandler(database, wsHub, publicURLOverride)") -if "reconHandler.CanaryPing" not in rt: - rt = rt.replace('r.Get("/recon/ping/{scan_id}", func(w http.ResponseWriter, r *http.Request) {\n\t\twriteJSON(w, map[string]interface{}{"ok": false, "error": "canary not configured"})\n\t})', 'r.Get("/recon/ping/{scan_id}", reconHandler.CanaryPing)') -wb("server/internal/api/router.go", rt) - -help = (ROOT / "server/web/src/help/settingHelp.ts").read_text(encoding="utf-8") -if "recon_form_fingerprint" not in help: - help = help.replace(" recon_deploy_kit:", " recon_form_fingerprint:\n 'Recon crawl scores form inputs by name/id/placeholder against SSRF-prone keywords (url, src, href, callback, redirect, avatar, import, feed, screenshot, pdf, proxy). Results include exact field name and id for operator paste targets in Deploy Recon.',\n recon_ssrf_canary:\n 'POST /api/v1/recon/scan with ssrf_canary:true registers a unique https://{public_host}/recon/ping/{scan_id} URL. Paste that URL into the scored field on the target; a hit confirms SSRF. Poll GET /api/v1/recon/canary/{scan_id} or watch recon_canary_hit on the dashboard WS.',\n recon_deploy_kit:") - wb("server/web/src/help/settingHelp.ts", help) -htest = (ROOT / "server/web/src/help/settingHelp.test.ts").read_text(encoding="utf-8") -if "recon_form_fingerprint" not in htest: - htest = htest.replace(" 'recon_deploy_kit',", " 'recon_deploy_kit',\n 'recon_form_fingerprint',\n 'recon_ssrf_canary',") - wb("server/web/src/help/settingHelp.test.ts", htest) - -env = {**dict(__import__("os").environ), "GOCACHE": str(ROOT / ".gocache")} -r = subprocess.run(["go", "test", "./internal/recon/...", "./internal/db/...", "./internal/api/...", "-count=1", "-run", "TestScoreFormField|TestExtractFormField|TestBuildSSRf|TestReconSSRf|TestReconScan"], cwd=str(SERVER), env=env) -if r.returncode != 0: - sys.exit(r.returncode) - -subprocess.run(["git", "pull", "origin", "main"], cwd=str(ROOT), check=True) -files = [ - "server/internal/recon/types.go", "server/internal/recon/fingerprint.go", "server/internal/recon/canary.go", - "server/internal/recon/crawl.go", "server/internal/recon/scan.go", "server/internal/recon/parse.go", - "server/internal/recon/recon_test.go", "server/internal/db/recon_canary.go", "server/internal/db/recon_canary_test.go", - "server/internal/api/recon_canary_hub.go", "server/internal/api/recon_handler.go", "server/internal/api/recon_handler_test.go", - "server/internal/api/router.go", "server/web/src/help/settingHelp.ts", "server/web/src/help/settingHelp.test.ts", - "scripts/_fingerprint.go.bak", "scripts/_recon_handler_canary.go.bak", "scripts/_recon_handler_test_canary.go.bak", - "scripts/install_batch1_atomic.py", -] -subprocess.run(["git", "add"] + files, cwd=str(ROOT), check=True) -subprocess.run(["git", "commit", "-m", "Add recon form fingerprint library and SSRF canary mode.", "-m", "Score crawl inputs by SSRF-prone field names and register ping-back canaries for operator paste confirmation."], cwd=str(ROOT), check=True) -subprocess.run(["git", "push", "origin", "main"], cwd=str(ROOT), check=True) -print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(ROOT), text=True).strip()) diff --git a/scripts/write-types.py b/scripts/write-types.py deleted file mode 100644 index 41e9b95..0000000 --- a/scripts/write-types.py +++ /dev/null @@ -1,101 +0,0 @@ -from pathlib import Path - -CONTENT = r"""package recon - -import "time" - -var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443} - -const ( - DefaultPortDialTimeout = 2 * time.Second - DefaultCrawlDepth = 2 - DefaultCrawlMaxPages = 50 -) - -type ScanRequest struct { - Host string `json:"host"` - Port int `json:"port,omitempty"` - Scheme string `json:"scheme,omitempty"` - Paths []string `json:"paths,omitempty"` - Profile string `json:"profile,omitempty"` -} - -type PortResult struct { - Port int `json:"port"` - Open bool `json:"open"` -} - -type FormFinding struct { - PageURL string `json:"page_url"` - Action string `json:"action,omitempty"` - Method string `json:"method,omitempty"` - Enctype string `json:"enctype,omitempty"` - Fields []string `json:"fields,omitempty"` - HasFile bool `json:"has_file_input,omitempty"` - Multipart bool `json:"multipart,omitempty"` -} - -type URLFieldFinding struct { - PageURL string `json:"page_url"` - Name string `json:"name"` - Type string `json:"type,omitempty"` - Hint string `json:"hint"` -} - -type PageFinding struct { - URL string `json:"url"` - StatusCode int `json:"status_code"` - Title string `json:"title,omitempty"` -} - -type CrawlReport struct { - PagesFetched int `json:"pages_fetched"` - Pages []PageFinding `json:"pages,omitempty"` - FileInputs []FormFinding `json:"file_inputs,omitempty"` - MultipartForms []FormFinding `json:"multipart_forms,omitempty"` - URLFields []URLFieldFinding `json:"url_fields,omitempty"` - SSRFScore int `json:"ssrf_score"` - CMSFingerprints []string `json:"cms_fingerprints,omitempty"` -} - -type DeployRecommendation struct { - Lane string `json:"lane,omitempty"` - Template string `json:"template,omitempty"` - Reason string `json:"reason"` - Priority int `json:"priority"` -} - -type ScanReport struct { - ScanID string `json:"scan_id,omitempty"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - Status string `json:"status,omitempty"` - ScannedAt time.Time `json:"scanned_at"` - Ports []PortResult `json:"ports"` - Crawl *CrawlReport `json:"crawl,omitempty"` - RelayVia string `json:"relay_via,omitempty"` - UDPHints []UDPHint `json:"udp_hints,omitempty"` - PathTracerHints []string `json:"path_tracer_hints,omitempty"` - Message string `json:"message,omitempty"` - Recommendations []DeployRecommendation `json:"recommendations,omitempty"` -} - -type ReconScanDiff struct { - NewPorts []int `json:"new_ports,omitempty"` - NewForms []FormFinding `json:"new_forms,omitempty"` -} - -type ReconHistoryEntry struct { - ScanID string `json:"scan_id"` - Host string `json:"host"` - Profile string `json:"profile,omitempty"` - Status string `json:"status"` - ScannedAt time.Time `json:"scanned_at"` - Report *ScanReport `json:"report,omitempty"` - Diff *ReconScanDiff `json:"diff,omitempty"` -} -""" - -Path(__file__).resolve().parents[1] / "server/internal/recon/types.go" -Path(r"g:/crypto miner/server/internal/recon/types.go").write_text(CONTENT, encoding="utf-8", newline="\n") -print("wrote types.go")