49c24e611c
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Score crawl inputs by SSRF-prone field names and register ping-back canaries for operator paste confirmation.
405 lines
18 KiB
Python
405 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(r"G:/crypto miner")
|
|
SERVER = ROOT / "server"
|
|
|
|
|
|
def wb(rel, text):
|
|
p = ROOT / rel
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
data = text.encode("utf-8")
|
|
if b"\x00" in data:
|
|
raise SystemExit("NUL in " + rel)
|
|
p.write_bytes(data)
|
|
|
|
|
|
def patch(rel, old, new):
|
|
p = ROOT / rel
|
|
t = p.read_text(encoding="utf-8")
|
|
if old not in t:
|
|
if new in t:
|
|
return
|
|
raise SystemExit("patch miss " + rel)
|
|
p.write_text(t.replace(old, new, 1), encoding="utf-8", newline="\n")
|
|
|
|
|
|
types = (ROOT / "scripts/install_batch1_final.py").read_text(encoding="utf-8")
|
|
# types block is embedded in install_batch1_final - write from known good merge
|
|
types = '''package recon
|
|
|
|
import "time"
|
|
|
|
var FleetPorts = []int{22, 80, 443, 445, 3389, 5985, 5986, 6262, 8080, 8443}
|
|
|
|
const (
|
|
DefaultPortDialTimeout = 2 * time.Second
|
|
DefaultCrawlDepth = 2
|
|
DefaultCrawlMaxPages = 50
|
|
)
|
|
|
|
type ScanRequest struct {
|
|
Host string `json:"host"`
|
|
Port int `json:"port,omitempty"`
|
|
Scheme string `json:"scheme,omitempty"`
|
|
Paths []string `json:"paths,omitempty"`
|
|
Profile string `json:"profile,omitempty"`
|
|
Profiles []string `json:"profiles,omitempty"`
|
|
SSRFCanary bool `json:"ssrf_canary,omitempty"`
|
|
}
|
|
|
|
type PortResult struct {
|
|
Port int `json:"port"`
|
|
Open bool `json:"open"`
|
|
}
|
|
|
|
type UDPHint struct {
|
|
Port int `json:"port"`
|
|
Open bool `json:"open"`
|
|
Service string `json:"service"`
|
|
}
|
|
|
|
type PortBanner struct {
|
|
Port int `json:"port"`
|
|
Service string `json:"service,omitempty"`
|
|
Banner string `json:"banner,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
Hint string `json:"hint,omitempty"`
|
|
}
|
|
|
|
type StackEntry struct {
|
|
Name string `json:"name"`
|
|
Source string `json:"source"`
|
|
Detail string `json:"detail,omitempty"`
|
|
}
|
|
|
|
type HTTPHeaderSnap struct {
|
|
URL string
|
|
Headers map[string]string
|
|
}
|
|
|
|
type FormFinding struct {
|
|
PageURL string `json:"page_url"`
|
|
Action string `json:"action,omitempty"`
|
|
Method string `json:"method,omitempty"`
|
|
Enctype string `json:"enctype,omitempty"`
|
|
Fields []string `json:"fields,omitempty"`
|
|
HasFile bool `json:"has_file_input,omitempty"`
|
|
Multipart bool `json:"multipart,omitempty"`
|
|
}
|
|
|
|
type URLFieldFinding struct {
|
|
PageURL string `json:"page_url"`
|
|
Name string `json:"name"`
|
|
ID string `json:"id,omitempty"`
|
|
Type string `json:"type,omitempty"`
|
|
Hint string `json:"hint"`
|
|
}
|
|
|
|
type FormFieldFingerprint struct {
|
|
PageURL string `json:"page_url"`
|
|
Name string `json:"name"`
|
|
ID string `json:"id,omitempty"`
|
|
Placeholder string `json:"placeholder,omitempty"`
|
|
Score int `json:"score"`
|
|
Matches []string `json:"matches,omitempty"`
|
|
PasteTarget string `json:"paste_target,omitempty"`
|
|
}
|
|
|
|
type SSRFCanaryInfo struct {
|
|
ScanID string `json:"scan_id"`
|
|
URL string `json:"url"`
|
|
Status string `json:"status"`
|
|
PasteTarget string `json:"paste_target"`
|
|
PasteFieldName string `json:"paste_field_name,omitempty"`
|
|
PasteFieldID string `json:"paste_field_id,omitempty"`
|
|
HitAt *time.Time `json:"hit_at,omitempty"`
|
|
}
|
|
|
|
type UploadHunterFinding struct {
|
|
PageURL string `json:"page_url"`
|
|
Target string `json:"target,omitempty"`
|
|
Source string `json:"source"`
|
|
Method string `json:"method,omitempty"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
Score int `json:"score"`
|
|
StatusCode int `json:"status_code,omitempty"`
|
|
}
|
|
|
|
type AdminSurfaceFinding struct {
|
|
Path string `json:"path"`
|
|
URL string `json:"url"`
|
|
StatusCode int `json:"status_code"`
|
|
Signal string `json:"signal"`
|
|
}
|
|
|
|
type PageFinding struct {
|
|
URL string `json:"url"`
|
|
StatusCode int `json:"status_code"`
|
|
Title string `json:"title,omitempty"`
|
|
}
|
|
|
|
type CrawlReport struct {
|
|
PagesFetched int `json:"pages_fetched"`
|
|
Pages []PageFinding `json:"pages,omitempty"`
|
|
FileInputs []FormFinding `json:"file_inputs,omitempty"`
|
|
MultipartForms []FormFinding `json:"multipart_forms,omitempty"`
|
|
URLFields []URLFieldFinding `json:"url_fields,omitempty"`
|
|
FingerprintFields []FormFieldFingerprint `json:"fingerprint_fields,omitempty"`
|
|
SSRFScore int `json:"ssrf_score"`
|
|
CMSFingerprints []string `json:"cms_fingerprints,omitempty"`
|
|
Stack []StackEntry `json:"stack,omitempty"`
|
|
UploadHunter []UploadHunterFinding `json:"upload_hunter,omitempty"`
|
|
}
|
|
|
|
type DeployRecommendation struct {
|
|
Lane string `json:"lane,omitempty"`
|
|
Template string `json:"template,omitempty"`
|
|
Reason string `json:"reason"`
|
|
Priority int `json:"priority"`
|
|
}
|
|
|
|
type ReconScanDiff struct {
|
|
NewPorts []int `json:"new_ports,omitempty"`
|
|
NewForms []FormFinding `json:"new_forms,omitempty"`
|
|
}
|
|
|
|
type ReconHistoryEntry struct {
|
|
ScanID string `json:"scan_id"`
|
|
Host string `json:"host"`
|
|
Profile string `json:"profile,omitempty"`
|
|
Status string `json:"status,omitempty"`
|
|
ScannedAt time.Time `json:"scanned_at"`
|
|
Report *ScanReport `json:"report"`
|
|
Diff *ReconScanDiff `json:"diff,omitempty"`
|
|
}
|
|
|
|
type ScanReport struct {
|
|
ScanID string `json:"scan_id,omitempty"`
|
|
Host string `json:"host"`
|
|
Profile string `json:"profile,omitempty"`
|
|
ProfilesUsed []string `json:"profiles_used,omitempty"`
|
|
Status string `json:"status,omitempty"`
|
|
ScannedAt time.Time `json:"scanned_at"`
|
|
Ports []PortResult `json:"ports"`
|
|
Banners []PortBanner `json:"banners,omitempty"`
|
|
Stack []StackEntry `json:"stack,omitempty"`
|
|
DeployKitLane string `json:"deploy_kit_lane,omitempty"`
|
|
Crawl *CrawlReport `json:"crawl,omitempty"`
|
|
AdminSurface []AdminSurfaceFinding `json:"admin_surface,omitempty"`
|
|
RelayVia string `json:"relay_via,omitempty"`
|
|
UDPHints []UDPHint `json:"udp_hints,omitempty"`
|
|
PathTracerHints []string `json:"path_tracer_hints,omitempty"`
|
|
Message string `json:"message,omitempty"`
|
|
Recommendations []DeployRecommendation `json:"recommendations,omitempty"`
|
|
Canary *SSRFCanaryInfo `json:"canary,omitempty"`
|
|
}
|
|
'''
|
|
|
|
stub = ROOT / "server/internal/recon/fingerprint_stub.go"
|
|
if stub.exists():
|
|
stub.unlink()
|
|
|
|
wb("server/internal/recon/types.go", types)
|
|
wb("server/internal/recon/fingerprint.go", (ROOT / "scripts/_fingerprint.go.bak").read_text(encoding="utf-8"))
|
|
wb("server/internal/recon/canary.go", (ROOT / "server/internal/recon/canary.go").read_text(encoding="utf-8") if (ROOT / "server/internal/recon/canary.go").exists() and "BuildSSRfCanaryURL" in (ROOT / "server/internal/recon/canary.go").read_text(encoding="utf-8") else '''package recon
|
|
|
|
import ("fmt"; "net/url"; "strings")
|
|
|
|
func BuildSSRfCanaryURL(publicBase, scanID string) string {
|
|
base := strings.TrimSpace(publicBase)
|
|
base = strings.TrimSuffix(base, "/")
|
|
if base == "" { base = "https://localhost" }
|
|
if strings.HasPrefix(base, "http://") { base = "https://" + strings.TrimPrefix(base, "http://") } else if !strings.HasPrefix(base, "https://") { base = "https://" + base }
|
|
u, err := url.Parse(base)
|
|
if err != nil { return fmt.Sprintf("https://localhost/recon/ping/%s", scanID) }
|
|
u.Scheme = "https"; u.Path = "/recon/ping/" + scanID; u.RawQuery = ""; u.Fragment = ""
|
|
return u.String()
|
|
}
|
|
''')
|
|
|
|
patch("server/internal/recon/parse.go", "Name: label,\n\t\t\t\tType:", "Name: label,\n\t\t\t\tID: id,\n\t\t\t\tType:")
|
|
for rel in ("server/internal/recon/crawl.go", "server/internal/recon/scan.go"):
|
|
patch(rel, "report.SSRFScore += pageScore\n", "report.SSRFScore += pageScore\n\t\tAppendPageFingerprints(report, item.url, body)\n")
|
|
|
|
wb("server/internal/db/recon_canary.go", (ROOT / "server/internal/db/recon_canary.go").read_text(encoding="utf-8") if (ROOT / "server/internal/db/recon_canary.go").exists() and "MarkReconSSRfCanaryHit" in (ROOT / "server/internal/db/recon_canary.go").read_text(encoding="utf-8") else Path(r"G:/crypto miner/server/internal/db/recon_canary.go").read_text(encoding="utf-8"))
|
|
|
|
# always write clean db if file missing key func - simpler: always overwrite from backup content in handler write above
|
|
# Re-read - use explicit content
|
|
wb("server/internal/db/recon_canary.go", '''package db
|
|
|
|
import (
|
|
"database/sql"
|
|
"time"
|
|
|
|
"crypto-miner-server/internal/recon"
|
|
)
|
|
|
|
func (d *Database) ensureReconSSRfCanaryTable() error {
|
|
_, err := d.Exec(`CREATE TABLE IF NOT EXISTS recon_ssrf_canary (
|
|
scan_id TEXT PRIMARY KEY, host TEXT NOT NULL, canary_url TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending', paste_field_name TEXT NOT NULL DEFAULT '',
|
|
paste_field_id TEXT NOT NULL DEFAULT '', hit_at DATETIME, created_at DATETIME NOT NULL)`)
|
|
return err
|
|
}
|
|
|
|
func (d *Database) InsertReconSSRfCanary(scanID, host, canaryURL, pasteName, pasteID string) error {
|
|
if err := d.ensureReconSSRfCanaryTable(); err != nil { return err }
|
|
_, err := d.Exec(`INSERT INTO recon_ssrf_canary (scan_id, host, canary_url, status, paste_field_name, paste_field_id, created_at) VALUES (?, ?, ?, 'pending', ?, ?, ?)`,
|
|
scanID, host, canaryURL, pasteName, pasteID, time.Now().UTC().Format(time.RFC3339Nano))
|
|
return err
|
|
}
|
|
|
|
func (d *Database) GetReconSSRfCanary(scanID string) (*recon.SSRFCanaryInfo, error) {
|
|
if err := d.ensureReconSSRfCanaryTable(); err != nil { return nil, err }
|
|
var host, canaryURL, status, pasteName, pasteID string
|
|
var hitAt sql.NullString
|
|
err := d.QueryRow(`SELECT host, canary_url, status, paste_field_name, paste_field_id, hit_at FROM recon_ssrf_canary WHERE scan_id = ?`, scanID).Scan(&host, &canaryURL, &status, &pasteName, &pasteID, &hitAt)
|
|
if err != nil { return nil, err }
|
|
info := &recon.SSRFCanaryInfo{ScanID: scanID, URL: canaryURL, Status: status, PasteTarget: canaryURL, PasteFieldName: pasteName, PasteFieldID: pasteID}
|
|
if hitAt.Valid { if ts, err := time.Parse(time.RFC3339Nano, hitAt.String); err == nil { info.HitAt = &ts } }
|
|
return info, nil
|
|
}
|
|
|
|
func (d *Database) UpdateReconSSRfCanaryPasteField(scanID, pasteName, pasteID string) error {
|
|
if err := d.ensureReconSSRfCanaryTable(); err != nil { return err }
|
|
_, err := d.Exec(`UPDATE recon_ssrf_canary SET paste_field_name = ?, paste_field_id = ? WHERE scan_id = ?`, pasteName, pasteID, scanID)
|
|
return err
|
|
}
|
|
|
|
func (d *Database) MarkReconSSRfCanaryHit(scanID string) (bool, error) {
|
|
if err := d.ensureReconSSRfCanaryTable(); err != nil { return false, err }
|
|
res, err := d.Exec(`UPDATE recon_ssrf_canary SET status = 'confirmed', hit_at = ? WHERE scan_id = ? AND status != 'confirmed'`, time.Now().UTC().Format(time.RFC3339Nano), scanID)
|
|
if err != nil { return false, err }
|
|
if n, _ := res.RowsAffected(); n > 0 { return true, nil }
|
|
var status string
|
|
err = d.QueryRow(`SELECT status FROM recon_ssrf_canary WHERE scan_id = ?`, scanID).Scan(&status)
|
|
return status == "confirmed", err
|
|
}
|
|
''')
|
|
|
|
wb("server/internal/db/recon_canary_test.go", '''package db
|
|
|
|
import "testing"
|
|
|
|
func TestReconSSRfCanaryInsertAndHit(t *testing.T) {
|
|
d, err := New(t.TempDir())
|
|
if err != nil { t.Fatal(err) }
|
|
t.Cleanup(func() { _ = d.Close() })
|
|
if err := d.InsertReconSSRfCanary("c1", "lab", "https://x/recon/ping/c1", "u", "i"); err != nil { t.Fatal(err) }
|
|
info, err := d.GetReconSSRfCanary("c1")
|
|
if err != nil || info.Status != "pending" { t.Fatal(info, err) }
|
|
hit, err := d.MarkReconSSRfCanaryHit("c1")
|
|
if err != nil || !hit { t.Fatal(hit, err) }
|
|
info, _ = d.GetReconSSRfCanary("c1")
|
|
if info.Status != "confirmed" || info.HitAt == nil { t.Fatal(info) }
|
|
}
|
|
''')
|
|
|
|
wb("server/internal/api/recon_canary_hub.go", '''package api
|
|
|
|
import ("sync"; "time"; "crypto-miner-server/internal/recon")
|
|
|
|
type ReconCanaryHub struct { mu sync.RWMutex; byID map[string]*recon.SSRFCanaryInfo }
|
|
|
|
func NewReconCanaryHub() *ReconCanaryHub { return &ReconCanaryHub{byID: map[string]*recon.SSRFCanaryInfo{}} }
|
|
|
|
func (h *ReconCanaryHub) Register(id string, info *recon.SSRFCanaryInfo) {
|
|
if h == nil || info == nil { return }
|
|
h.mu.Lock(); defer h.mu.Unlock(); cp := *info; h.byID[id] = &cp
|
|
}
|
|
|
|
func (h *ReconCanaryHub) Get(id string) *recon.SSRFCanaryInfo {
|
|
if h == nil { return nil }
|
|
h.mu.RLock(); defer h.mu.RUnlock()
|
|
if x := h.byID[id]; x != nil { cp := *x; return &cp }
|
|
return nil
|
|
}
|
|
|
|
func (h *ReconCanaryHub) MarkHit(id string) bool {
|
|
if h == nil { return false }
|
|
h.mu.Lock(); defer h.mu.Unlock()
|
|
x, ok := h.byID[id]; if !ok { return false }
|
|
if x.Status == "confirmed" { return true }
|
|
now := time.Now().UTC(); x.Status = "confirmed"; x.HitAt = &now; return true
|
|
}
|
|
|
|
func (h *ReconCanaryHub) UpdatePasteField(id, name, id2 string) {
|
|
if h == nil { return }
|
|
h.mu.Lock(); defer h.mu.Unlock()
|
|
if x := h.byID[id]; x != nil { x.PasteFieldName, x.PasteFieldID = name, id2 }
|
|
}
|
|
|
|
func BroadcastReconCanaryHit(hub *WSHub, scanID string) {
|
|
if hub == nil { return }
|
|
hub.broadcastDashboard(Message{Type: "recon_canary_hit", Payload: mustMarshal(map[string]interface{}{"scan_id": scanID, "status": "confirmed"})})
|
|
}
|
|
''')
|
|
|
|
wb("server/internal/api/recon_handler.go", (ROOT / "scripts/_recon_handler_canary.go.bak").read_text(encoding="utf-8"))
|
|
ht = (ROOT / "scripts/_recon_handler_test_canary.go.bak").read_text(encoding="utf-8")
|
|
if '"time"' not in ht:
|
|
ht = ht.replace('"testing"\n', '"testing"\n\t"time"\n')
|
|
wb("server/internal/api/recon_handler_test.go", ht)
|
|
|
|
rtest = (ROOT / "server/internal/recon/recon_test.go").read_text(encoding="utf-8")
|
|
if "TestBuildSSRfCanaryURLForcesHTTPS" not in rtest:
|
|
rtest += '''
|
|
|
|
func TestScoreFormFieldFingerprintKeywords(t *testing.T) {
|
|
score, matches := ScoreFormFieldFingerprint("callback_url", "", "")
|
|
if score < 14 || !containsStr(matches, "callback") { t.Fatalf("score=%d matches=%v", score, matches) }
|
|
}
|
|
|
|
func TestExtractFormFieldFingerprintsExactNameID(t *testing.T) {
|
|
fps := ExtractFormFieldFingerprints("http://lab/", `<input name="avatar_url" id="userAvatar">`)
|
|
if len(fps) == 0 || fps[0].Name != "avatar_url" || fps[0].ID != "userAvatar" { t.Fatalf("%+v", fps) }
|
|
}
|
|
|
|
func TestBuildSSRfCanaryURLForcesHTTPS(t *testing.T) {
|
|
u := BuildSSRfCanaryURL("http://public.example.com", "abc-123")
|
|
if !strings.HasPrefix(u, "https://") || !strings.Contains(u, "/recon/ping/abc-123") { t.Fatalf("url=%q", u) }
|
|
}
|
|
'''
|
|
wb("server/internal/recon/recon_test.go", rtest)
|
|
|
|
rt = (ROOT / "server/internal/api/router.go").read_text(encoding="utf-8")
|
|
if "recon/canary" not in rt:
|
|
rt = rt.replace('r.Post("/recon/scan", reconHandler.Scan)', 'r.Post("/recon/scan", reconHandler.Scan)\n\t\tr.Get("/recon/canary/{scan_id}", reconHandler.CanaryStatus)', 1)
|
|
if "NewReconHandler(database, wsHub, publicURLOverride)" not in rt:
|
|
rt = rt.replace("NewReconHandler(database, wsHub)", "NewReconHandler(database, wsHub, publicURLOverride)")
|
|
if "reconHandler.CanaryPing" not in rt:
|
|
rt = rt.replace('r.Get("/recon/ping/{scan_id}", func(w http.ResponseWriter, r *http.Request) {\n\t\twriteJSON(w, map[string]interface{}{"ok": false, "error": "canary not configured"})\n\t})', 'r.Get("/recon/ping/{scan_id}", reconHandler.CanaryPing)')
|
|
wb("server/internal/api/router.go", rt)
|
|
|
|
help = (ROOT / "server/web/src/help/settingHelp.ts").read_text(encoding="utf-8")
|
|
if "recon_form_fingerprint" not in help:
|
|
help = help.replace(" recon_deploy_kit:", " recon_form_fingerprint:\n 'Recon crawl scores form inputs by name/id/placeholder against SSRF-prone keywords (url, src, href, callback, redirect, avatar, import, feed, screenshot, pdf, proxy). Results include exact field name and id for operator paste targets in Deploy Recon.',\n recon_ssrf_canary:\n 'POST /api/v1/recon/scan with ssrf_canary:true registers a unique https://{public_host}/recon/ping/{scan_id} URL. Paste that URL into the scored field on the target; a hit confirms SSRF. Poll GET /api/v1/recon/canary/{scan_id} or watch recon_canary_hit on the dashboard WS.',\n recon_deploy_kit:")
|
|
wb("server/web/src/help/settingHelp.ts", help)
|
|
htest = (ROOT / "server/web/src/help/settingHelp.test.ts").read_text(encoding="utf-8")
|
|
if "recon_form_fingerprint" not in htest:
|
|
htest = htest.replace(" 'recon_deploy_kit',", " 'recon_deploy_kit',\n 'recon_form_fingerprint',\n 'recon_ssrf_canary',")
|
|
wb("server/web/src/help/settingHelp.test.ts", htest)
|
|
|
|
env = {**dict(__import__("os").environ), "GOCACHE": str(ROOT / ".gocache")}
|
|
r = subprocess.run(["go", "test", "./internal/recon/...", "./internal/db/...", "./internal/api/...", "-count=1", "-run", "TestScoreFormField|TestExtractFormField|TestBuildSSRf|TestReconSSRf|TestReconScan"], cwd=str(SERVER), env=env)
|
|
if r.returncode != 0:
|
|
sys.exit(r.returncode)
|
|
|
|
subprocess.run(["git", "pull", "origin", "main"], cwd=str(ROOT), check=True)
|
|
files = [
|
|
"server/internal/recon/types.go", "server/internal/recon/fingerprint.go", "server/internal/recon/canary.go",
|
|
"server/internal/recon/crawl.go", "server/internal/recon/scan.go", "server/internal/recon/parse.go",
|
|
"server/internal/recon/recon_test.go", "server/internal/db/recon_canary.go", "server/internal/db/recon_canary_test.go",
|
|
"server/internal/api/recon_canary_hub.go", "server/internal/api/recon_handler.go", "server/internal/api/recon_handler_test.go",
|
|
"server/internal/api/router.go", "server/web/src/help/settingHelp.ts", "server/web/src/help/settingHelp.test.ts",
|
|
"scripts/_fingerprint.go.bak", "scripts/_recon_handler_canary.go.bak", "scripts/_recon_handler_test_canary.go.bak",
|
|
"scripts/install_batch1_atomic.py",
|
|
]
|
|
subprocess.run(["git", "add"] + files, cwd=str(ROOT), check=True)
|
|
subprocess.run(["git", "commit", "-m", "Add recon form fingerprint library and SSRF canary mode.", "-m", "Score crawl inputs by SSRF-prone field names and register ping-back canaries for operator paste confirmation."], cwd=str(ROOT), check=True)
|
|
subprocess.run(["git", "push", "origin", "main"], cwd=str(ROOT), check=True)
|
|
print("COMMIT_HASH=" + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(ROOT), text=True).strip())
|