drjones/AetherForge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add recon upload hunter and admin surface probing for owned-target scans.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Details

Browse Source 
Extend web crawl with multipart/drag-drop/JS upload ranking and probe common admin paths for 200 vs 401/403 signals in scan JSON.
This commit is contained in:
AetherForge
2026-06-07 12:21:40 -07:00
parent 1fd3ec8618
commit 251bdfa1ac
5 changed files with 191 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
server/internal/recon/admin_surface.go
View File

@@ -8,11 +8,29 @@ import (
) )
var adminSurfacePaths = []string{ var adminSurfacePaths = []string{
"/wp-admin", "/wp-admin/", "/admin", "/admin/", "/admin/login", "/administrator", "/wp-admin",
"/api", "/api/", "/api/v1", "/graphql", "/graphql/", "/wp-admin/",
"/swagger", "/swagger/", "/swagger/index.html", "/swagger-ui", "/swagger-ui/", "/admin",
"/actuator", "/actuator/", "/actuator/health", "/admin/",
"/.env", "/.env.local", "/server-status", "/server-status/", "/admin/login",
"/administrator",
"/api",
"/api/",
"/api/v1",
"/graphql",
"/graphql/",
"/swagger",
"/swagger/",
"/swagger/index.html",
"/swagger-ui",
"/swagger-ui/",
"/actuator",
"/actuator/",
"/actuator/health",
"/.env",
"/.env.local",
"/server-status",
"/server-status/",
} }
func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFinding { func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFinding {
@@ -21,6 +39,7 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
port = defaultPortForScheme(scheme) port = defaultPortForScheme(scheme)
} }
base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port)) base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
seen := map[string]bool{} seen := map[string]bool{}
var out []AdminSurfaceFinding var out []AdminSurfaceFinding
for _, path := range adminSurfacePaths { for _, path := range adminSurfacePaths {
@@ -29,6 +48,7 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
continue continue
} }
seen[key] = true seen[key] = true
rawURL := strings.TrimRight(base, "/") + path rawURL := strings.TrimRight(base, "/") + path
status, _, _, err := fetchPage(rawURL) status, _, _, err := fetchPage(rawURL)
if err != nil { if err != nil {
@@ -38,8 +58,14 @@ func ProbeAdminSurface(host string, port int, scheme string) []AdminSurfaceFindi
if signal == "" { if signal == "" {
continue continue
} }
out = append(out, AdminSurfaceFinding{Path: path, URL: rawURL, StatusCode: status, Signal: signal}) out = append(out, AdminSurfaceFinding{
Path: path,
URL: rawURL,
StatusCode: status,
Signal: signal,
})
} }
sort.Slice(out, func(i, j int) bool { sort.Slice(out, func(i, j int) bool {
if out[i].Signal != out[j].Signal { if out[i].Signal != out[j].Signal {
return out[i].Signal == "green" return out[i].Signal == "green"

7
server/internal/recon/crawl.go
View File

@@ -25,6 +25,9 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
} }
report := &CrawlReport{} report := &CrawlReport{}
var uploadRaw []UploadHunterFinding
var jsQueue []string
jsSeen := map[string]bool{}
var headerSnaps []HTTPHeaderSnap var headerSnaps []HTTPHeaderSnap
var htmlBodies []string var htmlBodies []string
visited := map[string]bool{} visited := map[string]bool{}
@@ -65,6 +68,9 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
report.URLFields = append(report.URLFields, fields...) report.URLFields = append(report.URLFields, fields...)
report.SSRFScore += pageScore report.SSRFScore += pageScore
report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
uploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)
uploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)
collectUploadJSAtDepth(base, item.url, body, item.depth, DefaultCrawlDepth, jsSeen, &jsQueue)
if item.depth >= DefaultCrawlDepth { if item.depth >= DefaultCrawlDepth {
continue continue
@@ -89,6 +95,7 @@ func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlRepo
} }
report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
report.Stack = BuildStack(headerSnaps, htmlBodies) report.Stack = BuildStack(headerSnaps, htmlBodies)
report.UploadHunter = finalizeUploadHunter(base, uploadRaw, jsQueue)
return report, nil return report, nil
} }

110
server/internal/recon/recon_test.go
View File

@@ -162,6 +162,116 @@ func TestNormalizeOwnedHostRejectsEmpty(t *testing.T) {
} }
} }
func TestUploadHunterMultipartAndJS(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/":
w.Write([]byte(`<html><body><form action="/api/upload" enctype="multipart/form-data"><input type="file"></form><script src="/static/upload.js"></script></body></html>`))
case "/static/upload.js":
w.Write([]byte(`fetch("/api/upload")`))
case "/api/upload":
w.WriteHeader(http.StatusOK)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
port := 80
if p := u.Port(); p != "" {
port = atoi(p)
}
SetFetchPageHook(func(rawURL string) (int, string, error) {
resp, err := http.Get(rawURL)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
return resp.StatusCode, body, nil
})
t.Cleanup(func() { SetFetchPageHook(nil) })
report, err := Crawl(u.Hostname(), port, u.Scheme, []string{"/"})
if err != nil || len(report.UploadHunter) == 0 {
t.Fatalf("err=%v hunter=%+v", err, report.UploadHunter)
}
}
func TestProbeAdminSurfaceSignals(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/admin", "/admin/":
w.WriteHeader(http.StatusForbidden)
case "/swagger", "/swagger/":
w.WriteHeader(http.StatusOK)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
port := 80
if p := u.Port(); p != "" {
port = atoi(p)
}
SetFetchPageHook(func(rawURL string) (int, string, error) {
resp, err := http.Get(rawURL)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
return resp.StatusCode, body, nil
})
t.Cleanup(func() { SetFetchPageHook(nil) })
if len(ProbeAdminSurface(u.Hostname(), port, u.Scheme)) == 0 {
t.Fatal("expected admin surface")
}
}
func TestScanReportIncludesAdminSurfaceJSON(t *testing.T) {
SetPortDialHook(func(host string, port int, _ time.Duration) bool { return port == 80 })
t.Cleanup(func() { SetPortDialHook(nil) })
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/":
w.Write([]byte(`<html><form enctype="multipart/form-data" action="/api/upload"><input type="file"></form></html>`))
case "/api/upload":
w.WriteHeader(http.StatusOK)
case "/admin", "/admin/":
w.WriteHeader(http.StatusForbidden)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
port := 80
if p := u.Port(); p != "" {
port = atoi(p)
}
SetFetchPageHook(func(rawURL string) (int, string, error) {
resp, err := http.Get(rawURL)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
return resp.StatusCode, body, nil
})
t.Cleanup(func() { SetFetchPageHook(nil) })
report, err := Scan(ScanRequest{Host: u.Hostname(), Port: port, Scheme: u.Scheme})
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(report)
s := string(raw)
if !strings.Contains(s, `"admin_surface"`) || !strings.Contains(s, `"upload_hunter"`) {
t.Fatalf("json=%s", s)
}
}
func containsStr(list []string, want string) bool { func containsStr(list []string, want string) bool {
for _, s := range list { for _, s := range list {
if s == want { if s == want {

57
server/internal/recon/scan.go
View File

@@ -34,6 +34,7 @@ func scanLegacy(req ScanRequest) (*ScanReport, error) {
if err == nil && crawl != nil { if err == nil && crawl != nil {
report.Crawl = crawl report.Crawl = crawl
} }
report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
} }
report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false) report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false)
return report, nil return report, nil
@@ -127,7 +128,7 @@ func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackE
} }
if crawl != nil { if crawl != nil {
if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 { if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 || len(crawl.UploadHunter) > 0 {
recs = append(recs, DeployRecommendation{ recs = append(recs, DeployRecommendation{
Lane: "stage_fetch", Lane: "stage_fetch",
Reason: "Multipart or file-upload form — stage_fetch manifest staging", Reason: "Multipart or file-upload form — stage_fetch manifest staging",
@@ -183,9 +184,6 @@ func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanR
if !opts.SkipPorts { if !opts.SkipPorts {
ports := scanPortsList(host, portsToScan) ports := scanPortsList(host, portsToScan)
report.Ports = ports report.Ports = ports
for _, p := range ports {
emitReconPort(emit, scanID, host, p)
}
report.Banners = GrabBanners(host, ports) report.Banners = GrabBanners(host, ports)
} }
if shouldCrawlProfile(req, uxProfile, report.Ports, opts) { if shouldCrawlProfile(req, uxProfile, report.Ports, opts) {
@@ -194,6 +192,7 @@ func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanR
report.Crawl = crawl report.Crawl = crawl
report.Stack = MergeStack(crawl.Stack) report.Stack = MergeStack(crawl.Stack)
} }
report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
} }
report.DeployKitLane = SuggestDeployKitLane(report.Stack) report.DeployKitLane = SuggestDeployKitLane(report.Stack)
cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata) cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata)
@@ -245,6 +244,9 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
if maxPages <= 0 { maxPages = DefaultCrawlMaxPages } if maxPages <= 0 { maxPages = DefaultCrawlMaxPages }
if maxDepth < 0 { maxDepth = DefaultCrawlDepth } if maxDepth < 0 { maxDepth = DefaultCrawlDepth }
report := &CrawlReport{} report := &CrawlReport{}
var uploadRaw []UploadHunterFinding
var jsQueue []string
jsSeen := map[string]bool{}
var headerSnaps []HTTPHeaderSnap var headerSnaps []HTTPHeaderSnap
var htmlBodies []string var htmlBodies []string
visited := map[string]bool{} visited := map[string]bool{}
@@ -262,7 +264,6 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
report.PagesFetched++ report.PagesFetched++
title, _ := htmlParseTitle(body) title, _ := htmlParseTitle(body)
report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title}) report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})
emitReconPage(emit, scanID, host, item.url, status, title)
if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) } if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) }
htmlBodies = append(htmlBodies, body) htmlBodies = append(htmlBodies, body)
files, multi, fields, pageScore, cms := ParseHTML(item.url, body) files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
@@ -271,7 +272,9 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
report.URLFields = append(report.URLFields, fields...) report.URLFields = append(report.URLFields, fields...)
report.SSRFScore += pageScore report.SSRFScore += pageScore
report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms) report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
emitReconFindings(emit, scanID, host, files, multi, fields) uploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)
uploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)
collectUploadJSAtDepth(base, item.url, body, item.depth, maxDepth, jsSeen, &jsQueue)
if item.depth >= maxDepth { continue } if item.depth >= maxDepth { continue }
for _, link := range extractLinks(body) { for _, link := range extractLinks(body) {
abs, err := resolveSameOrigin(base, link) abs, err := resolveSameOrigin(base, link)
@@ -282,50 +285,10 @@ func crawlWithOptions(host string, port int, scheme string, seedPaths []string,
if report.SSRFScore > 100 { report.SSRFScore = 100 } if report.SSRFScore > 100 { report.SSRFScore = 100 }
report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints) report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
report.Stack = BuildStack(headerSnaps, htmlBodies) report.Stack = BuildStack(headerSnaps, htmlBodies)
report.UploadHunter = finalizeUploadHunter(base, uploadRaw, jsQueue)
return report, nil return report, nil
} }
func emitReconPort(emit StreamEmit, scanID, host string, p PortResult) {
if emit == nil {
return
}
emit("recon_port", map[string]interface{}{"scan_id": scanID, "host": host, "port": p.Port, "open": p.Open})
}
func emitReconPage(emit StreamEmit, scanID, host, pageURL string, status int, title string) {
if emit == nil {
return
}
emit("recon_page", map[string]interface{}{"scan_id": scanID, "host": host, "url": pageURL, "status_code": status, "title": title})
}
func emitReconFindings(emit StreamEmit, scanID, host string, files, multi []FormFinding, fields []URLFieldFinding) {
if emit == nil {
return
}
for _, f := range files {
emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "file_input", "finding": f})
}
for _, f := range multi {
emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "multipart_form", "finding": f})
}
for _, f := range fields {
emit("recon_finding", map[string]interface{}{"scan_id": scanID, "host": host, "kind": "url_field", "finding": f})
}
}
func formFindingKey(f FormFinding) string {
return f.PageURL + "|" + f.Action + "|" + strings.Join(f.Fields, ",")
}
func collectFormFindings(r *ScanReport) []FormFinding {
if r == nil || r.Crawl == nil {
return nil
}
out := append([]FormFinding{}, r.Crawl.FileInputs...)
return append(out, r.Crawl.MultipartForms...)
}
func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o } func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o }
func DiffReports(prev, cur *ScanReport) *ReconScanDiff { func DiffReports(prev, cur *ScanReport) *ReconScanDiff {
if cur == nil { return nil } if cur == nil { return nil }

39
server/internal/recon/upload_hunter.go
View File

@@ -19,13 +19,23 @@ func collectUploadFromPage(pageURL string, fileInputs, multipart []FormFinding)
if !f.HasFile { if !f.HasFile {
continue continue
} }
out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, f.Action), Source: "file_input", Method: f.Method}) out = append(out, UploadHunterFinding{
PageURL: pageURL,
Target: resolveUploadTarget(pageURL, f.Action),
Source: "file_input",
Method: f.Method,
})
} }
for _, f := range multipart { for _, f := range multipart {
if !f.Multipart { if !f.Multipart {
continue continue
} }
out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, f.Action), Source: "multipart", Method: f.Method}) out = append(out, UploadHunterFinding{
PageURL: pageURL,
Target: resolveUploadTarget(pageURL, f.Action),
Source: "multipart",
Method: f.Method,
})
} }
return out return out
} }
@@ -41,12 +51,17 @@ func detectDragDropZones(pageURL, body string) []UploadHunterFinding {
if n.tag != "" { if n.tag != "" {
cls := strings.ToLower(n.attr("class")) cls := strings.ToLower(n.attr("class"))
id := strings.ToLower(n.attr("id")) id := strings.ToLower(n.attr("id"))
if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || n.attr("data-dropzone") != "" { dropAttr := strings.ToLower(n.attr("data-dropzone"))
if dragDropClassRe.MatchString(cls) || dragDropClassRe.MatchString(id) || dropAttr != "" {
target := n.attr("data-upload-url") target := n.attr("data-upload-url")
if target == "" { if target == "" {
target = n.attr("action") target = n.attr("action")
} }
out = append(out, UploadHunterFinding{PageURL: pageURL, Target: resolveUploadTarget(pageURL, target), Source: "drag_drop"}) out = append(out, UploadHunterFinding{
PageURL: pageURL,
Target: resolveUploadTarget(pageURL, target),
Source: "drag_drop",
})
} }
} }
for _, c := range n.children { for _, c := range n.children {
@@ -62,10 +77,16 @@ func detectDragDropZones(pageURL, body string) []UploadHunterFinding {
func detectDragDropFromText(pageURL, body string) []UploadHunterFinding { func detectDragDropFromText(pageURL, body string) []UploadHunterFinding {
lower := strings.ToLower(body) lower := strings.ToLower(body)
if (!strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag")) || (!strings.Contains(lower, "upload") && !strings.Contains(lower, "file")) { if !strings.Contains(lower, "dropzone") && !strings.Contains(lower, "drag") {
return nil return nil
} }
return []UploadHunterFinding{{PageURL: pageURL, Source: "drag_drop"}} if !strings.Contains(lower, "upload") && !strings.Contains(lower, "file") {
return nil
}
return []UploadHunterFinding{{
PageURL: pageURL,
Source: "drag_drop",
}}
} }
func extractScriptSrc(pageURL, body string) []string { func extractScriptSrc(pageURL, body string) []string {
@@ -98,7 +119,11 @@ func scanJSForUpload(jsURL, body string) []UploadHunterFinding {
if m := openUploadPathRe.FindString(body); m != "" { if m := openUploadPathRe.FindString(body); m != "" {
target = m target = m
} }
return []UploadHunterFinding{{PageURL: jsURL, Target: target, Source: "js"}} return []UploadHunterFinding{{
PageURL: jsURL,
Target: target,
Source: "js",
}}
} }
func isScriptAsset(ref string) bool { func isScriptAsset(ref string) bool {