AetherForge/server/internal/recon/crawl.go

package recon

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
)

const maxHTMLBytes = 512 * 1024

// fetchPageFn overrides HTTP fetches in tests (nil = live GET from server host).
var fetchPageFn func(rawURL string) (status int, body string, err error)

// Crawl fetches seed URL and same-origin linked paths up to depth and maxPages.
func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {
	scheme = normalizeScheme(scheme, port)
	if port <= 0 {
		port = defaultPortForScheme(scheme)
	}
	base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
	seeds := seedPaths
	if len(seeds) == 0 {
		seeds = []string{"/"}
	}

	report := &CrawlReport{}
	visited := map[string]bool{}
	queue := []queuedURL{}
	for _, p := range seeds {
		abs, err := resolveSameOrigin(base, p)
		if err != nil {
			continue
		}
		queue = append(queue, queuedURL{url: abs, depth: 0})
	}

	for len(queue) > 0 && report.PagesFetched < DefaultCrawlMaxPages {
		item := queue[0]
		queue = queue[1:]
		key := normalizeURLKey(item.url)
		if visited[key] {
			continue
		}
		visited[key] = true

		status, body, err := fetchPage(item.url)
		if err != nil {
			continue
		}
		report.PagesFetched++
		title := ""
		if root, err := htmlParseTitle(body); err == nil {
			title = root
		}
		report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})

		files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
		report.FileInputs = append(report.FileInputs, files...)
		report.MultipartForms = append(report.MultipartForms, multi...)
		report.URLFields = append(report.URLFields, fields...)
		report.SSRFScore += pageScore
		report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)

		if item.depth >= DefaultCrawlDepth {
			continue
		}
		for _, link := range extractLinks(body) {
			abs, err := resolveSameOrigin(base, link)
			if err != nil {
				continue
			}
			if !sameOrigin(base, abs) {
				continue
			}
			lkey := normalizeURLKey(abs)
			if !visited[lkey] {
				queue = append(queue, queuedURL{url: abs, depth: item.depth + 1})
			}
		}
	}

	if report.SSRFScore > 100 {
		report.SSRFScore = 100
	}
	report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
	return report, nil
}

type queuedURL struct {
	url   string
	depth int
}

func fetchPage(rawURL string) (int, string, error) {
	if fetchPageFn != nil {
		return fetchPageFn(rawURL)
	}
	client := &http.Client{Timeout: DefaultPortDialTimeout}
	req, err := http.NewRequest(http.MethodGet, rawURL, nil)
	if err != nil {
		return 0, "", err
	}
	req.Header.Set("User-Agent", "AetherForge-Recon/1.0")
	resp, err := client.Do(req)
	if err != nil {
		return 0, "", err
	}
	defer resp.Body.Close()
	body, err := readBodyLimited(resp.Body, maxHTMLBytes)
	if err != nil {
		return resp.StatusCode, "", err
	}
	return resp.StatusCode, body, nil
}

func normalizeScheme(scheme string, port int) string {
	scheme = strings.ToLower(strings.TrimSpace(scheme))
	switch scheme {
	case "http", "https":
		return scheme
	}
	if port == 443 || port == 8443 {
		return "https"
	}
	return "http"
}

func defaultPortForScheme(scheme string) int {
	if scheme == "https" {
		return 443
	}
	return 80
}

func joinHostPort(host string, port int) string {
	if strings.Contains(host, ":") {
		return host
	}
	return fmt.Sprintf("%s:%d", host, port)
}

func resolveSameOrigin(base, ref string) (string, error) {
	baseURL, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return baseURL.String(), nil
	}
	refURL, err := url.Parse(ref)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(refURL).String(), nil
}

func sameOrigin(base, target string) bool {
	b, err := url.Parse(base)
	if err != nil {
		return false
	}
	t, err := url.Parse(target)
	if err != nil {
		return false
	}
	return strings.EqualFold(b.Scheme, t.Scheme) &&
		strings.EqualFold(b.Hostname(), t.Hostname()) &&
		b.Port() == t.Port()
}

func normalizeURLKey(raw string) string {
	u, err := url.Parse(raw)
	if err != nil {
		return strings.ToLower(strings.TrimSpace(raw))
	}
	u.Fragment = ""
	return strings.ToLower(u.String())
}

func extractLinks(body string) []string {
	root, err := htmlParseRoot(body)
	if err != nil {
		return nil
	}
	var links []string
	var walk func(*htmlNode)
	walk = func(n *htmlNode) {
		if n.tag == "a" {
			if href := n.attr("href"); href != "" && !strings.HasPrefix(strings.ToLower(href), "javascript:") {
				links = append(links, href)
			}
		}
		for _, c := range n.children {
			walk(c)
		}
	}
	walk(root)
	return links
}

func mergeCMS(existing, add []string) []string {
	seen := map[string]bool{}
	for _, tag := range existing {
		seen[tag] = true
	}
	var out []string
	out = append(out, existing...)
	for _, tag := range add {
		if tag == "" || seen[tag] {
			continue
		}
		seen[tag] = true
		out = append(out, tag)
	}
	return out
}

// htmlParseRoot and htmlParseTitle are thin wrappers to avoid exporting html types in tests.
func htmlParseRoot(body string) (*htmlNode, error) {
	root, err := parseHTMLTree(body)
	if err != nil {
		return nil, err
	}
	return toHTMLNode(root), nil
}

func htmlParseTitle(body string) (string, error) {
	root, err := parseHTMLTree(body)
	if err != nil {
		return "", err
	}
	return extractTitle(root), nil
}