Add browser deploy recon backend with port scan, web crawl, and deploy lane recommendations.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
POST /api/v1/recon/scan probes fleet ports from the server host, crawls owned HTTP targets, maps findings to spread lanes, and records optional oath ledger rows.
This commit is contained in:
235
server/internal/recon/crawl.go
Normal file
235
server/internal/recon/crawl.go
Normal file
@@ -0,0 +1,235 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxHTMLBytes = 512 * 1024
|
||||
|
||||
// fetchPageFn overrides HTTP fetches in tests (nil = live GET from server host).
|
||||
var fetchPageFn func(rawURL string) (status int, body string, err error)
|
||||
|
||||
// Crawl fetches seed URL and same-origin linked paths up to depth and maxPages.
|
||||
func Crawl(host string, port int, scheme string, seedPaths []string) (*CrawlReport, error) {
|
||||
scheme = normalizeScheme(scheme, port)
|
||||
if port <= 0 {
|
||||
port = defaultPortForScheme(scheme)
|
||||
}
|
||||
base := fmt.Sprintf("%s://%s", scheme, joinHostPort(host, port))
|
||||
seeds := seedPaths
|
||||
if len(seeds) == 0 {
|
||||
seeds = []string{"/"}
|
||||
}
|
||||
|
||||
report := &CrawlReport{}
|
||||
visited := map[string]bool{}
|
||||
queue := []queuedURL{}
|
||||
for _, p := range seeds {
|
||||
abs, err := resolveSameOrigin(base, p)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
queue = append(queue, queuedURL{url: abs, depth: 0})
|
||||
}
|
||||
|
||||
for len(queue) > 0 && report.PagesFetched < DefaultCrawlMaxPages {
|
||||
item := queue[0]
|
||||
queue = queue[1:]
|
||||
key := normalizeURLKey(item.url)
|
||||
if visited[key] {
|
||||
continue
|
||||
}
|
||||
visited[key] = true
|
||||
|
||||
status, body, err := fetchPage(item.url)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
report.PagesFetched++
|
||||
title := ""
|
||||
if root, err := htmlParseTitle(body); err == nil {
|
||||
title = root
|
||||
}
|
||||
report.Pages = append(report.Pages, PageFinding{URL: item.url, StatusCode: status, Title: title})
|
||||
|
||||
files, multi, fields, pageScore, cms := ParseHTML(item.url, body)
|
||||
report.FileInputs = append(report.FileInputs, files...)
|
||||
report.MultipartForms = append(report.MultipartForms, multi...)
|
||||
report.URLFields = append(report.URLFields, fields...)
|
||||
report.SSRFScore += pageScore
|
||||
report.CMSFingerprints = mergeCMS(report.CMSFingerprints, cms)
|
||||
|
||||
if item.depth >= DefaultCrawlDepth {
|
||||
continue
|
||||
}
|
||||
for _, link := range extractLinks(body) {
|
||||
abs, err := resolveSameOrigin(base, link)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !sameOrigin(base, abs) {
|
||||
continue
|
||||
}
|
||||
lkey := normalizeURLKey(abs)
|
||||
if !visited[lkey] {
|
||||
queue = append(queue, queuedURL{url: abs, depth: item.depth + 1})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if report.SSRFScore > 100 {
|
||||
report.SSRFScore = 100
|
||||
}
|
||||
report.CMSFingerprints = mergeCMS(nil, report.CMSFingerprints)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
type queuedURL struct {
|
||||
url string
|
||||
depth int
|
||||
}
|
||||
|
||||
func fetchPage(rawURL string) (int, string, error) {
|
||||
if fetchPageFn != nil {
|
||||
return fetchPageFn(rawURL)
|
||||
}
|
||||
client := &http.Client{Timeout: DefaultPortDialTimeout}
|
||||
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "AetherForge-Recon/1.0")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := readBodyLimited(resp.Body, maxHTMLBytes)
|
||||
if err != nil {
|
||||
return resp.StatusCode, "", err
|
||||
}
|
||||
return resp.StatusCode, body, nil
|
||||
}
|
||||
|
||||
func normalizeScheme(scheme string, port int) string {
|
||||
scheme = strings.ToLower(strings.TrimSpace(scheme))
|
||||
switch scheme {
|
||||
case "http", "https":
|
||||
return scheme
|
||||
}
|
||||
if port == 443 || port == 8443 {
|
||||
return "https"
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
|
||||
func defaultPortForScheme(scheme string) int {
|
||||
if scheme == "https" {
|
||||
return 443
|
||||
}
|
||||
return 80
|
||||
}
|
||||
|
||||
func joinHostPort(host string, port int) string {
|
||||
if strings.Contains(host, ":") {
|
||||
return host
|
||||
}
|
||||
return fmt.Sprintf("%s:%d", host, port)
|
||||
}
|
||||
|
||||
func resolveSameOrigin(base, ref string) (string, error) {
|
||||
baseURL, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return baseURL.String(), nil
|
||||
}
|
||||
refURL, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return baseURL.ResolveReference(refURL).String(), nil
|
||||
}
|
||||
|
||||
func sameOrigin(base, target string) bool {
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t, err := url.Parse(target)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(b.Scheme, t.Scheme) &&
|
||||
strings.EqualFold(b.Hostname(), t.Hostname()) &&
|
||||
b.Port() == t.Port()
|
||||
}
|
||||
|
||||
func normalizeURLKey(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return strings.ToLower(strings.TrimSpace(raw))
|
||||
}
|
||||
u.Fragment = ""
|
||||
return strings.ToLower(u.String())
|
||||
}
|
||||
|
||||
func extractLinks(body string) []string {
|
||||
root, err := htmlParseRoot(body)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var links []string
|
||||
var walk func(*htmlNode)
|
||||
walk = func(n *htmlNode) {
|
||||
if n.tag == "a" {
|
||||
if href := n.attr("href"); href != "" && !strings.HasPrefix(strings.ToLower(href), "javascript:") {
|
||||
links = append(links, href)
|
||||
}
|
||||
}
|
||||
for _, c := range n.children {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return links
|
||||
}
|
||||
|
||||
func mergeCMS(existing, add []string) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, tag := range existing {
|
||||
seen[tag] = true
|
||||
}
|
||||
var out []string
|
||||
out = append(out, existing...)
|
||||
for _, tag := range add {
|
||||
if tag == "" || seen[tag] {
|
||||
continue
|
||||
}
|
||||
seen[tag] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// htmlParseRoot and htmlParseTitle are thin wrappers to avoid exporting html types in tests.
|
||||
func htmlParseRoot(body string) (*htmlNode, error) {
|
||||
root, err := parseHTMLTree(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return toHTMLNode(root), nil
|
||||
}
|
||||
|
||||
func htmlParseTitle(body string) (string, error) {
|
||||
root, err := parseHTMLTree(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return extractTitle(root), nil
|
||||
}
|
||||
13
server/internal/recon/hooks.go
Normal file
13
server/internal/recon/hooks.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package recon
|
||||
|
||||
import "time"
|
||||
|
||||
// SetPortDialHook installs a test hook for TCP dials; pass nil to restore live dials.
|
||||
func SetPortDialHook(fn func(host string, port int, timeout time.Duration) bool) {
|
||||
dialPortFn = fn
|
||||
}
|
||||
|
||||
// SetFetchPageHook installs a test hook for HTTP fetches; pass nil to restore live GETs.
|
||||
func SetFetchPageHook(fn func(rawURL string) (status int, body string, err error)) {
|
||||
fetchPageFn = fn
|
||||
}
|
||||
40
server/internal/recon/html_tree.go
Normal file
40
server/internal/recon/html_tree.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type htmlNode struct {
|
||||
tag string
|
||||
attrs map[string]string
|
||||
children []*htmlNode
|
||||
}
|
||||
|
||||
func (n *htmlNode) attr(key string) string {
|
||||
if n == nil || n.attrs == nil {
|
||||
return ""
|
||||
}
|
||||
return n.attrs[key]
|
||||
}
|
||||
|
||||
func parseHTMLTree(body string) (*html.Node, error) {
|
||||
return html.Parse(strings.NewReader(body))
|
||||
}
|
||||
|
||||
func toHTMLNode(n *html.Node) *htmlNode {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
out := &htmlNode{tag: n.Data, attrs: map[string]string{}}
|
||||
for _, a := range n.Attr {
|
||||
out.attrs[a.Key] = a.Val
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if child := toHTMLNode(c); child != nil {
|
||||
out.children = append(out.children, child)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
219
server/internal/recon/parse.go
Normal file
219
server/internal/recon/parse.go
Normal file
@@ -0,0 +1,219 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var urlFieldHints = []string{
|
||||
"url", "link", "preview", "webhook", "fetch", "import", "oembed", "image",
|
||||
}
|
||||
|
||||
var cmsPathMarkers = []struct {
|
||||
path string
|
||||
tag string
|
||||
}{
|
||||
{"/wp-admin", "wordpress"},
|
||||
{"/wp-content", "wordpress"},
|
||||
{"/strapi", "strapi"},
|
||||
{"/graphql", "graphql"},
|
||||
{"/admin/login", "admin_login"},
|
||||
}
|
||||
|
||||
// ParseHTML extracts upload forms, URL fields, SSRF score, and CMS hints from HTML.
|
||||
func ParseHTML(pageURL, body string) (fileInputs, multipart []FormFinding, urlFields []URLFieldFinding, ssrfScore int, cms []string) {
|
||||
root, err := html.Parse(strings.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, nil, nil, 0, cmsFromText(pageURL, body)
|
||||
}
|
||||
title := extractTitle(root)
|
||||
_ = title
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.Data == "form" {
|
||||
form := parseForm(pageURL, n)
|
||||
if form.HasFile {
|
||||
fileInputs = append(fileInputs, form)
|
||||
}
|
||||
if form.Multipart {
|
||||
multipart = append(multipart, form)
|
||||
}
|
||||
ssrfScore += scoreForm(form)
|
||||
}
|
||||
if n.Type == html.ElementNode && n.Data == "input" {
|
||||
if field := parseURLField(pageURL, n); field != nil {
|
||||
urlFields = append(urlFields, *field)
|
||||
ssrfScore += 10
|
||||
}
|
||||
}
|
||||
if n.Type == html.ElementNode && (n.Data == "textarea" || n.Data == "select") {
|
||||
if field := parseURLFieldFromNamed(pageURL, attr(n, "name"), attr(n, "id"), attr(n, "placeholder")); field != nil {
|
||||
urlFields = append(urlFields, *field)
|
||||
ssrfScore += 8
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
|
||||
cms = cmsFromText(pageURL, body)
|
||||
if ssrfScore > 100 {
|
||||
ssrfScore = 100
|
||||
}
|
||||
return fileInputs, multipart, urlFields, ssrfScore, cms
|
||||
}
|
||||
|
||||
func parseForm(pageURL string, form *html.Node) FormFinding {
|
||||
f := FormFinding{
|
||||
PageURL: pageURL,
|
||||
Action: attr(form, "action"),
|
||||
Method: strings.ToLower(attr(form, "method")),
|
||||
Enctype: strings.ToLower(attr(form, "enctype")),
|
||||
}
|
||||
if f.Method == "" {
|
||||
f.Method = "get"
|
||||
}
|
||||
if strings.Contains(f.Enctype, "multipart") {
|
||||
f.Multipart = true
|
||||
}
|
||||
for c := form.FirstChild; c != nil; c = c.NextSibling {
|
||||
collectFormFields(c, &f)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func collectFormFields(n *html.Node, f *FormFinding) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "input", "textarea", "select":
|
||||
name := attr(n, "name")
|
||||
if name != "" {
|
||||
f.Fields = append(f.Fields, name)
|
||||
}
|
||||
if n.Data == "input" && strings.EqualFold(attr(n, "type"), "file") {
|
||||
f.HasFile = true
|
||||
f.Multipart = true
|
||||
}
|
||||
if field := parseURLField(f.PageURL, n); field != nil {
|
||||
f.Fields = append(f.Fields, field.Name+"("+field.Hint+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
collectFormFields(c, f)
|
||||
}
|
||||
}
|
||||
|
||||
func parseURLField(pageURL string, n *html.Node) *URLFieldFinding {
|
||||
name := attr(n, "name")
|
||||
id := attr(n, "id")
|
||||
placeholder := attr(n, "placeholder")
|
||||
return parseURLFieldFromNamed(pageURL, name, id, placeholder)
|
||||
}
|
||||
|
||||
func parseURLFieldFromNamed(pageURL, name, id, placeholder string) *URLFieldFinding {
|
||||
joined := strings.ToLower(strings.Join([]string{name, id, placeholder}, " "))
|
||||
for _, hint := range urlFieldHints {
|
||||
if strings.Contains(joined, hint) {
|
||||
label := name
|
||||
if label == "" {
|
||||
label = id
|
||||
}
|
||||
return &URLFieldFinding{
|
||||
PageURL: pageURL,
|
||||
Name: label,
|
||||
Type: "text",
|
||||
Hint: hint,
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scoreForm(f FormFinding) int {
|
||||
score := 0
|
||||
action := strings.ToLower(f.Action)
|
||||
for _, hint := range urlFieldHints {
|
||||
if strings.Contains(action, hint) {
|
||||
score += 15
|
||||
}
|
||||
}
|
||||
for _, field := range f.Fields {
|
||||
lower := strings.ToLower(field)
|
||||
for _, hint := range urlFieldHints {
|
||||
if strings.Contains(lower, hint) {
|
||||
score += 5
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.HasFile {
|
||||
score += 5
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func cmsFromText(pageURL, body string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
lowerURL := strings.ToLower(pageURL)
|
||||
lowerBody := strings.ToLower(body)
|
||||
add := func(tag string) {
|
||||
if tag == "" || seen[tag] {
|
||||
return
|
||||
}
|
||||
seen[tag] = true
|
||||
out = append(out, tag)
|
||||
}
|
||||
for _, marker := range cmsPathMarkers {
|
||||
if strings.Contains(lowerURL, marker.path) || strings.Contains(lowerBody, marker.path) {
|
||||
add(marker.tag)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lowerBody, "strapi") {
|
||||
add("strapi")
|
||||
}
|
||||
if strings.Contains(lowerBody, "wp-content") || strings.Contains(lowerBody, "wordpress") {
|
||||
add("wordpress")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractTitle(root *html.Node) string {
|
||||
var title string
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
|
||||
title = strings.TrimSpace(n.FirstChild.Data)
|
||||
return
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return title
|
||||
}
|
||||
|
||||
func attr(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func readBodyLimited(r io.Reader, max int64) (string, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
_, err := io.Copy(buf, io.LimitReader(r, max))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
32
server/internal/recon/portscan.go
Normal file
32
server/internal/recon/portscan.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dialPortFn overrides TCP probes in tests (nil = live dial from server host).
|
||||
var dialPortFn func(host string, port int, timeout time.Duration) bool
|
||||
|
||||
// ScanPorts TCP-dials common fleet ports on host with timeout from server host.
|
||||
func ScanPorts(host string) []PortResult {
|
||||
out := make([]PortResult, 0, len(FleetPorts))
|
||||
for _, port := range FleetPorts {
|
||||
out = append(out, PortResult{Port: port, Open: dialPort(host, port, DefaultPortDialTimeout)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dialPort(host string, port int, timeout time.Duration) bool {
|
||||
if dialPortFn != nil {
|
||||
return dialPortFn(host, port, timeout)
|
||||
}
|
||||
addr := net.JoinHostPort(host, strconv.Itoa(port))
|
||||
conn, err := net.DialTimeout("tcp", addr, timeout)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
186
server/internal/recon/recon_test.go
Normal file
186
server/internal/recon/recon_test.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestScanPortsWithInject(t *testing.T) {
|
||||
SetPortDialHook(func(host string, port int, _ time.Duration) bool {
|
||||
if host != "10.0.0.5" {
|
||||
t.Fatalf("host=%q", host)
|
||||
}
|
||||
return port == 22 || port == 443
|
||||
})
|
||||
t.Cleanup(func() { SetPortDialHook(nil) })
|
||||
results := ScanPorts("10.0.0.5")
|
||||
open := map[int]bool{}
|
||||
for _, r := range results {
|
||||
if r.Open {
|
||||
open[r.Port] = true
|
||||
}
|
||||
}
|
||||
if !open[22] || !open[443] {
|
||||
t.Fatalf("open=%v", open)
|
||||
}
|
||||
if open[445] {
|
||||
t.Fatal("445 should be closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLFixtures(t *testing.T) {
|
||||
body := `<html><head><title>Upload</title></head><body>
|
||||
<form action="/import" method="post" enctype="multipart/form-data">
|
||||
<input type="file" name="payload">
|
||||
<input type="text" name="webhook_url" value="">
|
||||
</form>
|
||||
<a href="/admin/login">Admin</a>
|
||||
<link href="/wp-content/themes/x/style.css">
|
||||
</body></html>`
|
||||
files, multi, fields, score, cms := ParseHTML("http://lab/upload", body)
|
||||
if len(files) != 1 || !files[0].HasFile {
|
||||
t.Fatalf("file inputs: %+v", files)
|
||||
}
|
||||
if len(multi) != 1 || !multi[0].Multipart {
|
||||
t.Fatalf("multipart: %+v", multi)
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
t.Fatal("expected url fields")
|
||||
}
|
||||
if score < 10 {
|
||||
t.Fatalf("ssrf score=%d", score)
|
||||
}
|
||||
if !containsStr(cms, "wordpress") && !containsStr(cms, "admin_login") {
|
||||
t.Fatalf("cms=%v", cms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawlSameOriginDepth(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
w.Write([]byte(`<html><a href="/page2">next</a><a href="http://evil.example/x">off</a></html>`))
|
||||
case "/page2":
|
||||
w.Write([]byte(`<html><a href="/page3">deep</a></html>`))
|
||||
case "/page3":
|
||||
w.Write([]byte(`<html>leaf</html>`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := 80
|
||||
if p := u.Port(); p != "" {
|
||||
port = atoi(p)
|
||||
}
|
||||
|
||||
SetFetchPageHook(func(rawURL string) (int, string, error) {
|
||||
resp, err := http.Get(rawURL)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := readBodyLimited(resp.Body, maxHTMLBytes)
|
||||
return resp.StatusCode, body, nil
|
||||
})
|
||||
t.Cleanup(func() { SetFetchPageHook(nil) })
|
||||
|
||||
report, err := Crawl(u.Hostname(), port, u.Scheme, []string{"/"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.PagesFetched < 2 {
|
||||
t.Fatalf("pages=%d", report.PagesFetched)
|
||||
}
|
||||
for _, p := range report.Pages {
|
||||
if strings.Contains(p.URL, "evil.example") {
|
||||
t.Fatalf("followed off-origin %s", p.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanReportRecommendations(t *testing.T) {
|
||||
ports := []PortResult{
|
||||
{Port: 22, Open: true},
|
||||
{Port: 5985, Open: true},
|
||||
{Port: 80, Open: true},
|
||||
}
|
||||
crawl := &CrawlReport{
|
||||
MultipartForms: []FormFinding{{PageURL: "http://x/", Multipart: true}},
|
||||
SSRFScore: 40,
|
||||
}
|
||||
recs := BuildRecommendations(ports, crawl)
|
||||
if len(recs) < 5 {
|
||||
t.Fatalf("recs=%+v", recs)
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, r := range recs {
|
||||
keys[r.Lane+"|"+r.Template] = true
|
||||
}
|
||||
for _, want := range []string{"linux_lotl|linux-lotl", "winrm|winrm", "bits_curl|", "stage_fetch|", "|ssrf_probe", "|public_waterhole"} {
|
||||
if !keys[want] {
|
||||
t.Fatalf("missing %q in %+v", want, recs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanOwnedTarget(t *testing.T) {
|
||||
SetPortDialHook(func(host string, port int, _ time.Duration) bool {
|
||||
return port == 80
|
||||
})
|
||||
t.Cleanup(func() { SetPortDialHook(nil) })
|
||||
SetFetchPageHook(func(rawURL string) (int, string, error) {
|
||||
return 200, `<html><form enctype="multipart/form-data"><input type="file" name="f"></form></html>`, nil
|
||||
})
|
||||
t.Cleanup(func() { SetFetchPageHook(nil) })
|
||||
report, err := Scan(ScanRequest{Host: "owned.lab", Port: 80, Scheme: "http"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Host != "owned.lab" {
|
||||
t.Fatalf("host=%q", report.Host)
|
||||
}
|
||||
if report.Crawl == nil || len(report.Recommendations) == 0 {
|
||||
raw, _ := json.Marshal(report)
|
||||
t.Fatalf("report=%s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeOwnedHostRejectsEmpty(t *testing.T) {
|
||||
if _, err := Scan(ScanRequest{}); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func containsStr(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func atoi(s string) int {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return 80
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
if n == 0 {
|
||||
return 80
|
||||
}
|
||||
return n
|
||||
}
|
||||
152
server/internal/recon/scan.go
Normal file
152
server/internal/recon/scan.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package recon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Scan runs port scan and optional web crawl for an operator-supplied owned target.
|
||||
func Scan(req ScanRequest) (*ScanReport, error) {
|
||||
host, err := normalizeOwnedHost(req.Host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ports := ScanPorts(host)
|
||||
report := &ScanReport{
|
||||
Host: host,
|
||||
ScannedAt: time.Now().UTC(),
|
||||
Ports: ports,
|
||||
}
|
||||
|
||||
if shouldCrawl(req, ports) {
|
||||
crawl, err := Crawl(host, req.Port, req.Scheme, req.Paths)
|
||||
if err == nil && crawl != nil {
|
||||
report.Crawl = crawl
|
||||
}
|
||||
}
|
||||
report.Recommendations = BuildRecommendations(ports, report.Crawl)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func normalizeOwnedHost(host string) (string, error) {
|
||||
host = strings.TrimSpace(host)
|
||||
if host == "" {
|
||||
return "", fmt.Errorf("host required")
|
||||
}
|
||||
host = strings.TrimPrefix(host, "http://")
|
||||
host = strings.TrimPrefix(host, "https://")
|
||||
if i := strings.Index(host, "/"); i >= 0 {
|
||||
host = host[:i]
|
||||
}
|
||||
if h, p, err := net.SplitHostPort(host); err == nil {
|
||||
if strings.TrimSpace(h) == "" {
|
||||
return "", fmt.Errorf("invalid host")
|
||||
}
|
||||
_ = p
|
||||
host = h
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return ip.String(), nil
|
||||
}
|
||||
if len(host) > 253 || strings.Contains(host, " ") {
|
||||
return "", fmt.Errorf("invalid host")
|
||||
}
|
||||
return strings.ToLower(host), nil
|
||||
}
|
||||
|
||||
func shouldCrawl(req ScanRequest, ports []PortResult) bool {
|
||||
if req.Port > 0 || strings.TrimSpace(req.Scheme) != "" || len(req.Paths) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, p := range ports {
|
||||
switch p.Port {
|
||||
case 80, 443, 8080, 8443:
|
||||
if p.Open {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BuildRecommendations maps port and crawl findings to existing deploy lanes/templates.
|
||||
func BuildRecommendations(ports []PortResult, crawl *CrawlReport) []DeployRecommendation {
|
||||
var recs []DeployRecommendation
|
||||
open := map[int]bool{}
|
||||
for _, p := range ports {
|
||||
if p.Open {
|
||||
open[p.Port] = true
|
||||
}
|
||||
}
|
||||
|
||||
if open[22] {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Lane: "linux_lotl",
|
||||
Template: "linux-lotl",
|
||||
Reason: "TCP 22 open — SSH LOTL bootstrap",
|
||||
Priority: 25,
|
||||
})
|
||||
}
|
||||
if open[445] {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Lane: "spread_smb_unc",
|
||||
Reason: "TCP 445 open — SMB UNC spread",
|
||||
Priority: 50,
|
||||
})
|
||||
}
|
||||
if open[5985] || open[5986] {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Lane: "winrm",
|
||||
Template: "winrm",
|
||||
Reason: "TCP 5985/5986 open — WinRM bootstrap",
|
||||
Priority: 30,
|
||||
})
|
||||
}
|
||||
if open[80] || open[443] || open[8080] || open[8443] {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Lane: "bits_curl",
|
||||
Reason: "HTTP surface open — dropper curl|bash one-liner",
|
||||
Priority: 20,
|
||||
})
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Template: "public_waterhole",
|
||||
Reason: "HTTP surface open — copy /spread/ public waterhole landing",
|
||||
Priority: 15,
|
||||
})
|
||||
}
|
||||
|
||||
if crawl != nil {
|
||||
if len(crawl.FileInputs) > 0 || len(crawl.MultipartForms) > 0 {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Lane: "stage_fetch",
|
||||
Reason: "Multipart or file-upload form — stage_fetch manifest staging",
|
||||
Priority: 35,
|
||||
})
|
||||
}
|
||||
if crawl.SSRFScore >= 30 {
|
||||
recs = append(recs, DeployRecommendation{
|
||||
Template: "ssrf_probe",
|
||||
Reason: fmt.Sprintf("SSRF candidate score %d — probe URL/webhook fields", crawl.SSRFScore),
|
||||
Priority: 45,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return dedupeRecommendations(recs)
|
||||
}
|
||||
|
||||
func dedupeRecommendations(in []DeployRecommendation) []DeployRecommendation {
|
||||
seen := map[string]bool{}
|
||||
var out []DeployRecommendation
|
||||
for _, r := range in {
|
||||
key := r.Lane + "|" + r.Template
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
80
server/internal/recon/types.go
Normal file
80
server/internal/recon/types.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package recon
|
||||
|
||||
import "time"
|
||||
|
||||
// FleetPorts are TCP ports probed during browser-deploy recon.
|
||||
var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443}
|
||||
|
||||
const (
|
||||
DefaultPortDialTimeout = 2 * time.Second
|
||||
DefaultCrawlDepth = 2
|
||||
DefaultCrawlMaxPages = 50
|
||||
)
|
||||
|
||||
// ScanRequest is operator-supplied owned-target input.
|
||||
type ScanRequest struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Scheme string `json:"scheme,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// PortResult is one TCP dial outcome.
|
||||
type PortResult struct {
|
||||
Port int `json:"port"`
|
||||
Open bool `json:"open"`
|
||||
}
|
||||
|
||||
// FormFinding describes an HTML form of interest.
|
||||
type FormFinding struct {
|
||||
PageURL string `json:"page_url"`
|
||||
Action string `json:"action,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Enctype string `json:"enctype,omitempty"`
|
||||
Fields []string `json:"fields,omitempty"`
|
||||
HasFile bool `json:"has_file_input,omitempty"`
|
||||
Multipart bool `json:"multipart,omitempty"`
|
||||
}
|
||||
|
||||
// URLFieldFinding is an input/textarea whose name or label hints URL fetch behavior.
|
||||
type URLFieldFinding struct {
|
||||
PageURL string `json:"page_url"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Hint string `json:"hint"`
|
||||
}
|
||||
|
||||
// PageFinding summarizes one crawled page.
|
||||
type PageFinding struct {
|
||||
URL string `json:"url"`
|
||||
StatusCode int `json:"status_code"`
|
||||
Title string `json:"title,omitempty"`
|
||||
}
|
||||
|
||||
// CrawlReport aggregates web surface findings.
|
||||
type CrawlReport struct {
|
||||
PagesFetched int `json:"pages_fetched"`
|
||||
Pages []PageFinding `json:"pages,omitempty"`
|
||||
FileInputs []FormFinding `json:"file_inputs,omitempty"`
|
||||
MultipartForms []FormFinding `json:"multipart_forms,omitempty"`
|
||||
URLFields []URLFieldFinding `json:"url_fields,omitempty"`
|
||||
SSRFScore int `json:"ssrf_score"`
|
||||
CMSFingerprints []string `json:"cms_fingerprints,omitempty"`
|
||||
}
|
||||
|
||||
// DeployRecommendation maps recon findings to an existing spread/deploy lane or template.
|
||||
type DeployRecommendation struct {
|
||||
Lane string `json:"lane,omitempty"`
|
||||
Template string `json:"template,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
Priority int `json:"priority"`
|
||||
}
|
||||
|
||||
// ScanReport is the full owned-target recon payload returned by POST /api/v1/recon/scan.
|
||||
type ScanReport struct {
|
||||
Host string `json:"host"`
|
||||
ScannedAt time.Time `json:"scanned_at"`
|
||||
Ports []PortResult `json:"ports"`
|
||||
Crawl *CrawlReport `json:"crawl,omitempty"`
|
||||
Recommendations []DeployRecommendation `json:"recommendations,omitempty"`
|
||||
}
|
||||
Reference in New Issue
Block a user