Add browser deploy recon backend with port scan, web crawl, and deploy lane recommendations.

POST /api/v1/recon/scan probes fleet ports from the server host, crawls owned HTTP targets, maps findings to spread lanes, and records optional oath ledger rows.

server/internal/recon/crawl.go

@@ -0,0 +1,235 @@
+package recon
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+const maxHTMLBytes = 512 * 1024
+
+// fetchPageFn overrides HTTP fetches in tests (nil = live GET from server host).
+var fetchPageFn func(rawURL string) (status int, body string, err error)
+
+// Crawl fetches seed URL and same-origin linked paths up to depth and maxPages.
+func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {
+	scheme = normalizeScheme(scheme, port)
+	if port <= 0 {
+		port = defaultPortForScheme(scheme)
+	}
+	base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
+	seeds := seedPaths
+	if len(seeds) == 0 {
+		seeds = []string{"/"}
+	}
+
+	report := &CrawlReport{}
+	visited := map[string]bool{}
+	queue := []queuedURL{}
+	for _, p := range seeds {
+		abs, err := resolveSameOrigin(base, p)
+		if err != nil {
+			continue
+		}
+		queue = append(queue, queuedURL{url: abs, depth: 0})
+	}
+
+	for len(queue) > 0 && report.PagesFetched < DefaultCrawlMaxPages {
+		item := queue[0]
+		queue = queue[1:]
+		key := normalizeURLKey(item.url)
+		if visited[key] {
+			continue
+		}
+		visited[key] = true
+
+		status, body, err := fetchPage(item.url)
+		if err != nil {
+			continue
+		}
+		report.PagesFetched++
+		title := ""
+		if root, err := htmlParseTitle(body); err == nil {
+			title = root
+		}
+		report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})
+
+		files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
+		report.FileInputs = append(report.FileInputs, files...)
+		report.MultipartForms = append(report.MultipartForms, multi...)
+		report.URLFields = append(report.URLFields, fields...)
+		report.SSRFScore += pageScore
+		report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
+
+		if item.depth >= DefaultCrawlDepth {
+			continue
+		}
+		for _, link := range extractLinks(body) {
+			abs, err := resolveSameOrigin(base, link)
+			if err != nil {
+				continue
+			}
+			if !sameOrigin(base, abs) {
+				continue
+			}
+			lkey := normalizeURLKey(abs)
+			if !visited[lkey] {
+				queue = append(queue, queuedURL{url: abs, depth: item.depth + 1})
+			}
+		}
+	}
+
+	if report.SSRFScore > 100 {
+		report.SSRFScore = 100
+	}
+	report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
+	return report, nil
+}
+
+type queuedURL struct {
+	url   string
+	depth int
+}
+
+func fetchPage(rawURL string) (int, string, error) {
+	if fetchPageFn != nil {
+		return fetchPageFn(rawURL)
+	}
+	client := &http.Client{Timeout: DefaultPortDialTimeout}
+	req, err := http.NewRequest(http.MethodGet, rawURL, nil)
+	if err != nil {
+		return 0, "", err
+	}
+	req.Header.Set("User-Agent", "AetherForge-Recon/1.0")
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close()
+	body, err := readBodyLimited(resp.Body, maxHTMLBytes)
+	if err != nil {
+		return resp.StatusCode, "", err
+	}
+	return resp.StatusCode, body, nil
+}
+
+func normalizeScheme(scheme string, port int) string {
+	scheme = strings.ToLower(strings.TrimSpace(scheme))
+	switch scheme {
+	case "http", "https":
+		return scheme
+	}
+	if port == 443 || port == 8443 {
+		return "https"
+	}
+	return "http"
+}
+
+func defaultPortForScheme(scheme string) int {
+	if scheme == "https" {
+		return 443
+	}
+	return 80
+}
+
+func joinHostPort(host string, port int) string {
+	if strings.Contains(host, ":") {
+		return host
+	}
+	return fmt.Sprintf("%s:%d", host, port)
+}
+
+func resolveSameOrigin(base, ref string) (string, error) {
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return "", err
+	}
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return baseURL.String(), nil
+	}
+	refURL, err := url.Parse(ref)
+	if err != nil {
+		return "", err
+	}
+	return baseURL.ResolveReference(refURL).String(), nil
+}
+
+func sameOrigin(base, target string) bool {
+	b, err := url.Parse(base)
+	if err != nil {
+		return false
+	}
+	t, err := url.Parse(target)
+	if err != nil {
+		return false
+	}
+	return strings.EqualFold(b.Scheme, t.Scheme) &&
+		strings.EqualFold(b.Hostname(), t.Hostname()) &&
+		b.Port() == t.Port()
+}
+
+func normalizeURLKey(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return strings.ToLower(strings.TrimSpace(raw))
+	}
+	u.Fragment = ""
+	return strings.ToLower(u.String())
+}
+
+func extractLinks(body string) []string {
+	root, err := htmlParseRoot(body)
+	if err != nil {
+		return nil
+	}
+	var links []string
+	var walk func(*htmlNode)
+	walk = func(n *htmlNode) {
+		if n.tag == "a" {
+			if href := n.attr("href"); href != "" && !strings.HasPrefix(strings.ToLower(href), "javascript:") {
+				links = append(links, href)
+			}
+		}
+		for _, c := range n.children {
+			walk(c)
+		}
+	}
+	walk(root)
+	return links
+}
+
+func mergeCMS(existing, add []string) []string {
+	seen := map[string]bool{}
+	for _, tag := range existing {
+		seen[tag] = true
+	}
+	var out []string
+	out = append(out, existing...)
+	for _, tag := range add {
+		if tag == "" || seen[tag] {
+			continue
+		}
+		seen[tag] = true
+		out = append(out, tag)
+	}
+	return out
+}
+
+// htmlParseRoot and htmlParseTitle are thin wrappers to avoid exporting html types in tests.
+func htmlParseRoot(body string) (*htmlNode, error) {
+	root, err := parseHTMLTree(body)
+	if err != nil {
+		return nil, err
+	}
+	return toHTMLNode(root), nil
+}
+
+func htmlParseTitle(body string) (string, error) {
+	root, err := parseHTMLTree(body)
+	if err != nil {
+		return "", err
+	}
+	return extractTitle(root), nil
+}