Add recon upload hunter and admin surface probing for owned-target scans.

Extend web crawl with multipart/drag-drop/JS upload ranking and probe common admin paths for 200 vs 401/403 signals in scan JSON.
2026-06-07 12:21:40 -07:00
parent 1fd3ec8618
commit 251bdfa1ac
5 changed files with 191 additions and 60 deletions
--- a/server/internal/recon/admin_surface.go
+++ b/server/internal/recon/admin_surface.go
@@ -8,11 +8,29 @@ import (
 )

 var adminSurfacePaths = []string{
-	"/wp-admin", "/wp-admin/", "/admin", "/admin/", "/admin/login", "/administrator",
-	"/api", "/api/", "/api/v1", "/graphql", "/graphql/",
-	"/swagger", "/swagger/", "/swagger/index.html", "/swagger-ui", "/swagger-ui/",
-	"/actuator", "/actuator/", "/actuator/health",
-	"/.env", "/.env.local", "/server-status", "/server-status/",
+	"/wp-admin",
+	"/wp-admin/",
+	"/admin",
+	"/admin/",
+	"/admin/login",
+	"/administrator",
+	"/api",
+	"/api/",
+	"/api/v1",
+	"/graphql",
+	"/graphql/",
+	"/swagger",
+	"/swagger/",
+	"/swagger/index.html",
+	"/swagger-ui",
+	"/swagger-ui/",
+	"/actuator",
+	"/actuator/",
+	"/actuator/health",
+	"/.env",
+	"/.env.local",
+	"/server-status",
+	"/server-status/",
 }

 func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFinding {
@@ -21,6 +39,7 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
 		port = defaultPortForScheme(scheme)
 	}
 	base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
+
 	seen := map[string]bool{}
 	var out []AdminSurfaceFinding
 	for _, path := range adminSurfacePaths {
@@ -29,6 +48,7 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
 			continue
 		}
 		seen[key] = true
+
 		rawURL := strings.TrimRight(base, "/") + path
 		status, _, _, err := fetchPage(rawURL)
 		if err != nil {
@@ -38,8 +58,14 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
 		if signal == "" {
 			continue
 		}
-		out = append(out, AdminSurfaceFinding{Path: path, URL: rawURL, StatusCode: status, Signal: signal})
+		out = append(out, AdminSurfaceFinding{
+			Path:       path,
+			URL:        rawURL,
+			StatusCode: status,
+			Signal:     signal,
+		})
 	}
+
 	sort.Slice(out, func(i, j int) bool {
 		if out[i].Signal != out[j].Signal {
 			return out[i].Signal == "green"
--- a/server/internal/recon/crawl.go
+++ b/server/internal/recon/crawl.go
@@ -25,6 +25,9 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
 	}

 	report := &CrawlReport{}
+	var uploadRaw []UploadHunterFinding
+	var jsQueue []string
+	jsSeen := map[string]bool{}
 	var headerSnaps []HTTPHeaderSnap
 	var htmlBodies []string
 	visited := map[string]bool{}
@@ -65,6 +68,9 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
 		report.URLFields = append(report.URLFields, fields...)
 		report.SSRFScore += pageScore
 		report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
+		uploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)
+		uploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)
+		collectUploadJSAtDepth(base, item.url, body, item.depth, DefaultCrawlDepth, jsSeen, &jsQueue)

 		if item.depth >= DefaultCrawlDepth {
 			continue
@@ -89,6 +95,7 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
 	}
 	report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
 	report.Stack = BuildStack(headerSnaps, htmlBodies)
+	report.UploadHunter = finalizeUploadHunter(base, uploadRaw, jsQueue)
 	return report, nil
 }

--- a/server/internal/recon/recon_test.go
+++ b/server/internal/recon/recon_test.go
@@ -162,6 +162,116 @@ func TestNormalizeOwnedHostRejectsEmpty(t *testing.T) {
 	}
 }

+
+func TestUploadHunterMultipartAndJS(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/":
+			w.Write([]byte(`<html><body><form action="/api/upload" enctype="multipart/form-data"><input type="file"></form><script src="/static/upload.js"></script></body></html>`))
+		case "/static/upload.js":
+			w.Write([]byte(`fetch("/api/upload")`))
+		case "/api/upload":
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+	port := 80
+	if p := u.Port(); p != "" {
+		port = atoi(p)
+	}
+	SetFetchPageHook(func(rawURL string) (int, string, error) {
+		resp, err := http.Get(rawURL)
+		if err != nil {
+			return 0, "", err
+		}
+		defer resp.Body.Close()
+		body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
+		return resp.StatusCode, body, nil
+	})
+	t.Cleanup(func() { SetFetchPageHook(nil) })
+	report, err := Crawl(u.Hostname(), port, u.Scheme, []string{"/"})
+	if err != nil || len(report.UploadHunter) == 0 {
+		t.Fatalf("err=%v hunter=%+v", err, report.UploadHunter)
+	}
+}
+
+func TestProbeAdminSurfaceSignals(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/admin", "/admin/":
+			w.WriteHeader(http.StatusForbidden)
+		case "/swagger", "/swagger/":
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+	port := 80
+	if p := u.Port(); p != "" {
+		port = atoi(p)
+	}
+	SetFetchPageHook(func(rawURL string) (int, string, error) {
+		resp, err := http.Get(rawURL)
+		if err != nil {
+			return 0, "", err
+		}
+		defer resp.Body.Close()
+		body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
+		return resp.StatusCode, body, nil
+	})
+	t.Cleanup(func() { SetFetchPageHook(nil) })
+	if len(ProbeAdminSurface(u.Hostname(), port, u.Scheme)) == 0 {
+		t.Fatal("expected admin surface")
+	}
+}
+
+func TestScanReportIncludesAdminSurfaceJSON(t *testing.T) {
+	SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 80 })
+	t.Cleanup(func() { SetPortDialHook(nil) })
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/":
+			w.Write([]byte(`<html><form enctype="multipart/form-data" action="/api/upload"><input type="file"></form></html>`))
+		case "/api/upload":
+			w.WriteHeader(http.StatusOK)
+		case "/admin", "/admin/":
+			w.WriteHeader(http.StatusForbidden)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+	port := 80
+	if p := u.Port(); p != "" {
+		port = atoi(p)
+	}
+	SetFetchPageHook(func(rawURL string) (int, string, error) {
+		resp, err := http.Get(rawURL)
+		if err != nil {
+			return 0, "", err
+		}
+		defer resp.Body.Close()
+		body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
+		return resp.StatusCode, body, nil
+	})
+	t.Cleanup(func() { SetFetchPageHook(nil) })
+	report, err := Scan(ScanRequest{Host: u.Hostname(), Port: port, Scheme: u.Scheme})
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, _ := json.Marshal(report)
+	s := string(raw)
+	if !strings.Contains(s, `"admin_surface"`) || !strings.Contains(s, `"upload_hunter"`) {
+		t.Fatalf("json=%s", s)
+	}
+}
+
 func containsStr(list []string, want string) bool {
 	for _, s := range list {
 		if s == want {
--- a/server/internal/recon/scan.go
+++ b/server/internal/recon/scan.go
@@ -34,6 +34,7 @@ func scanLegacy(req ScanRequest) (*ScanReport, error) {
 		if err == nil && crawl != nil {
 			report.Crawl = crawl
 		}
+		report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
 	}
 	report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false)
 	return report, nil
@@ -127,7 +128,7 @@ func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackE
 	}

 	if crawl != nil {
-		if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 {
+		if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 || len(crawl.UploadHunter) > 0 {
 			recs = append(recs, DeployRecommendation{
 				Lane:   "stage_fetch",
 				Reason: "Multipart or file-upload form â€” stage_fetch manifest staging",
@@ -183,9 +184,6 @@ func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanR
 	if !opts.SkipPorts {
 		ports := scanPortsList(host, portsToScan)
 		report.Ports = ports
-		for _, p := range ports {
-			emitReconPort(emit, scanID, host, p)
-		}
 		report.Banners = GrabBanners(host, ports)
 	}
 	if shouldCrawlProfile(req, uxProfile, report.Ports, opts) {
@@ -194,6 +192,7 @@ func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanR
 			report.Crawl = crawl
 			report.Stack = MergeStack(crawl.Stack)
 		}
+		report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
 	}
 	report.DeployKitLane = SuggestDeployKitLane(report.Stack)
 	cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata)
@@ -245,6 +244,9 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
 	if maxPages <= 0 { maxPages = DefaultCrawlMaxPages }
 	if maxDepth < 0 { maxDepth = DefaultCrawlDepth }
 	report := &CrawlReport{}
+	var uploadRaw []UploadHunterFinding
+	var jsQueue []string
+	jsSeen := map[string]bool{}
 	var headerSnaps []HTTPHeaderSnap
 	var htmlBodies []string
 	visited := map[string]bool{}
@@ -262,7 +264,6 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
 		report.PagesFetched++
 		title, _ := htmlParseTitle(body)
 		report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})
-		emitReconPage(emit, scanID, host, item.url, status, title)
 		if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) }
 		htmlBodies = append(htmlBodies, body)
 		files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
@@ -271,7 +272,9 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
 		report.URLFields = append(report.URLFields, fields...)
 		report.SSRFScore += pageScore
 		report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
-		emitReconFindings(emit, scanID, host, files, multi, fields)
+		uploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)
+		uploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)
+		collectUploadJSAtDepth(base, item.url, body, item.depth, maxDepth, jsSeen, &jsQueue)
 		if item.depth >= maxDepth { continue }
 		for _, link := range extractLinks(body) {
 			abs, err := resolveSameOrigin(base, link)
@@ -282,50 +285,10 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
 	if report.SSRFScore > 100 { report.SSRFScore = 100 }
 	report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
 	report.Stack = BuildStack(headerSnaps, htmlBodies)
+	report.UploadHunter = finalizeUploadHunter(base, uploadRaw, jsQueue)
 	return report, nil
 }

-func emitReconPort(emit StreamEmit, scanID, host string, p PortResult) {
-	if emit == nil {
-		return
-	}
-	emit("recon_port", map[string]interface{}{"scan_id": scanID, "host": host, "port": p.Port, "open": p.Open})
-}
-
-func emitReconPage(emit StreamEmit, scanID, host, pageURL string, status int, title string) {
-	if emit == nil {
-		return
-	}
-	emit("recon_page", map[string]interface{}{"scan_id": scanID, "host": host, "url": pageURL, "status_code": status, "title": title})
-}
-
-func emitReconFindings(emit StreamEmit, scanID, host string, files, multi []FormFinding, fields []URLFieldFinding) {
-	if emit == nil {
-		return
-	}
-	for _, f := range files {
-		emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "file_input", "finding": f})
-	}
-	for _, f := range multi {
-		emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "multipart_form", "finding": f})
-	}
-	for _, f := range fields {
-		emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "url_field", "finding": f})
-	}
-}
-
-func formFindingKey(f FormFinding) string {
-	return f.PageURL + "|" + f.Action + "|" + strings.Join(f.Fields, ",")
-}
-
-func collectFormFindings(r *ScanReport) []FormFinding {
-	if r == nil || r.Crawl == nil {
-		return nil
-	}
-	out := append([]FormFinding{}, r.Crawl.FileInputs...)
-	return append(out, r.Crawl.MultipartForms...)
-}
-
 func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o }
 func DiffReports(prev, cur *ScanReport) *ReconScanDiff {
 	if cur == nil { return nil }
--- a/server/internal/recon/upload_hunter.go
+++ b/server/internal/recon/upload_hunter.go
@@ -19,13 +19,23 @@ func collectUploadFromPage(pageURL string, fileInputs, multipart []FormFinding)
 		if !f.HasFile {
 			continue
 		}
-		out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, f.Action), Source: "file_input", Method: f.Method})
+		out = append(out, UploadHunterFinding{
+			PageURL: pageURL,
+			Target:  resolveUploadTarget(pageURL, f.Action),
+			Source:  "file_input",
+			Method:  f.Method,
+		})
 	}
 	for _, f := range multipart {
 		if !f.Multipart {
 			continue
 		}
-		out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, f.Action), Source: "multipart", Method: f.Method})
+		out = append(out, UploadHunterFinding{
+			PageURL: pageURL,
+			Target:  resolveUploadTarget(pageURL, f.Action),
+			Source:  "multipart",
+			Method:  f.Method,
+		})
 	}
 	return out
 }
@@ -41,12 +51,17 @@ func detectDragDropZones(pageURL, body string) []UploadHunterFinding {
 		if n.tag != "" {
 			cls := strings.ToLower(n.attr("class"))
 			id := strings.ToLower(n.attr("id"))
-			if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || n.attr("data-dropzone") != "" {
+			dropAttr := strings.ToLower(n.attr("data-dropzone"))
+			if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || dropAttr != "" {
 				target := n.attr("data-upload-url")
 				if target == "" {
 					target = n.attr("action")
 				}
-				out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, target), Source: "drag_drop"})
+				out = append(out, UploadHunterFinding{
+					PageURL: pageURL,
+					Target:  resolveUploadTarget(pageURL, target),
+					Source:  "drag_drop",
+				})
 			}
 		}
 		for _, c := range n.children {
@@ -62,10 +77,16 @@ func detectDragDropZones(pageURL, body string) []UploadHunterFinding {

 func detectDragDropFromText(pageURL, body string) []UploadHunterFinding {
 	lower := strings.ToLower(body)
-	if (!strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag")) || (!strings.Contains(lower, "upload") && !strings.Contains(lower, "file")) {
+	if !strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag") {
 		return nil
 	}
-	return []UploadHunterFinding{{PageURL: pageURL, Source: "drag_drop"}}
+	if !strings.Contains(lower, "upload") && !strings.Contains(lower, "file") {
+		return nil
+	}
+	return []UploadHunterFinding{{
+		PageURL: pageURL,
+		Source:  "drag_drop",
+	}}
 }

 func extractScriptSrc(pageURL, body string) []string {
@@ -98,7 +119,11 @@ func scanJSForUpload(jsURL, body string) []UploadHunterFinding {
 	if m := openUploadPathRe.FindString(body); m != "" {
 		target = m
 	}
-	return []UploadHunterFinding{{PageURL: jsURL, Target: target, Source: "js"}}
+	return []UploadHunterFinding{{
+		PageURL: jsURL,
+		Target:  target,
+		Source:  "js",
+	}}
 }

 func isScriptAsset(ref string) bool {