th-analyzer/engine/hunter.sh

#!/usr/bin/env bash
# ============================================================
# HUNTER — Mass Vulnerability Discovery Engine
# Part of The Analyzer
# Finds vulnerable websites at scale using OSINT + nuclei
# ============================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ANALYZER_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
REPORTS_DIR="$ANALYZER_DIR/reports"
TARGETS_DIR="$ANALYZER_DIR/targets"
WORK_DIR="/tmp/analyzer-hunter"
NUCLEI_TEMPLATES="${NUCLEI_TEMPLATES:-$HOME/nuclei-templates}"
HTTPS="$HOME/go/bin/httpx"

mkdir -p "$WORK_DIR" "$REPORTS_DIR" "$TARGETS_DIR"

# Colors
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
BLUE='\033[0;34m'; NC='\033[0m'

log() { echo -e "${BLUE}[*]${NC} $1"; }
ok()  { echo -e "${GREEN}[✓]${NC} $1"; }
warn(){ echo -e "${YELLOW}[!]${NC} $1"; }
err() { echo -e "${RED}[✗]${NC} $1"; }

# ============================================================
# PHASE 1: TARGET DISCOVERY
# ============================================================

discover_from_tranco() {
  local count="${1:-500}"
  local output="$TARGETS_DIR/hunt_tranco.txt"

  log "Fetching top $count sites from Tranco..."
  curl -skL "https://tranco-list.eu/top-1m.csv.zip" -o "$WORK_DIR/top1m.zip" 2>/dev/null

  if unzip -o "$WORK_DIR/top1m.zip" -d "$WORK_DIR" 2>/dev/null; then
    head -"$count" "$WORK_DIR"/top-1m.csv 2>/dev/null | cut -d, -f2 > "$output"
    ok "$(wc -l < "$output") domains from Tranco"
    return 0
  fi
  return 1
}

discover_from_analyzer() {
  local output="$TARGETS_DIR/hunt_analyzer.txt"
  if [ -f "$TARGETS_DIR/top50.txt" ]; then
    cp "$TARGETS_DIR/top50.txt" "$output"
    ok "$(wc -l < "$output") from Analyzer target list"
    return 0
  fi
  return 1
}

discover_vulnerable_software() {
  local output="$WORK_DIR/vuln_software_targets.txt"
  log "Building vulnerable software target list..."

  # Sites running known-vulnerable software
  cat > "$WORK_DIR/vuln_sites.txt" << 'VULNSITES'
# Known software vendors/instances that may have vulnerable versions
wordpress.org
joomla.org
drupal.org
magento.com
prestashop.com
opencart.com
phpmyadmin.net
roundcube.net
cpanel.net
php.net
apache.org
nginx.org
mysql.com
postgresql.org
mongodb.com
nodejs.org
laravel.com
symfony.com
rails.org
docker.com
kubernetes.io
jenkins.io
gitlab.com
sonarqube.org
grafana.com
prometheus.io
elastic.co
redis.io
tomcat.apache.org
jira.atlassian.com
confluence.atlassian.com
vbforum.com
simplemachines.org
phpbb.com
mediawiki.org
VULNSITES

  # Also add common CMS plugin repositories
  echo "woocommerce.com" >> "$WORK_DIR/vuln_sites.txt"
  echo "easy-digital-downloads.com" >> "$WORK_DIR/vuln_sites.txt"

  httpx -l "$WORK_DIR/vuln_sites.txt" -rl 20 -silent -timeout 5 \
    -o "$output" 2>/dev/null

  if [ -s "$output" ]; then
    ok "$(wc -l < "$output") live software targets"
  fi
}

# ============================================================
# PHASE 2: TECH DETECTION & TARGETED CVE SCANNING
# ============================================================

detect_technologies() {
  local targets_file="$1"
  local output="$WORK_DIR/tech_detected.json"

  log "Detecting technologies on $(wc -l < "$targets_file") targets..."
  httpx -l "$targets_file" -tech-detect -j -rl 20 -silent -timeout 5 \
    -o "$output" 2>/dev/null

  local count=$(wc -l < "$output" 2>/dev/null || echo 0)
  ok "Tech detected on $count hosts"

  # Summary
  python3 -c "
import json, sys
from collections import Counter
techs = Counter()
with open('$output') as f:
    for line in f:
        try:
            d = json.loads(line)
            for t in d.get('tech', []):
                techs[t] += 1
        except: pass
print('Top technologies detected:')
for tech, cnt in techs.most_common(20):
    print(f'  {tech}: {cnt}')
" 2>/dev/null

  echo "$output"
}

run_focused_cve_scan() {
  local targets_file="$1"
  local tech_file="$2"
  local output="$REPORTS_DIR/hunter_cve_$(date +%Y%m%d_%H%M%S).json"

  # Map technologies to specific CVE template categories
  log "Running focused CVE scan..."

  # Scan ALL targets with general CVE templates (faster than all 4k)
  nuclei -l "$targets_file" \
    -j \
    -rl 30 \
    -c 15 \
    -t "$NUCLEI_TEMPLATES/http/cves/" \
    -o "$output" \
    -severity critical,high \
    -silent \
    -stats \
    2>/dev/null

  if [ -s "$output" ]; then
    ok "$(wc -l < "$output") critical/high findings"
  else
    warn "No critical/high findings in mass scan"
  fi

  # Also run medium + exploitation templates for more depth
  local output2="$REPORTS_DIR/hunter_cve_medium_$(date +%s).json"
  nuclei -l "$targets_file" \
    -j \
    -rl 20 \
    -c 10 \
    -t "$NUCLEI_TEMPLATES/http/cves/" \
    -o "$output2" \
    -severity medium \
    -silent \
    2>/dev/null

  echo "$output"
}

# ============================================================
# PHASE 3: THE ANALYZER INTEGRATION
# ============================================================

analyze_findings() {
  local findings_file="$1"
  local output="$REPORTS_DIR/hunter_report_$(date +%Y%m%d_%H%M).md"

  log "Generating report..."

  python3 -c "
import json, sys
from collections import Counter
from datetime import datetime

findings = []
cves = Counter()
severities = Counter()

with open('$findings_file') as f:
    for line in f:
        try:
            d = json.loads(line)
            sev = d.get('info',{}).get('severity','unknown')
            severities[sev] += 1

            # Extract CVE IDs
            cve_list = []
            for ref in d.get('info',{}).get('classification',{}).get('cve',[]):
                cve_list.append(ref.get('id',''))
            if not cve_list:
                # Try alternative CVE sources
                for ref in d.get('info',{}).get('reference',[]):
                    if 'cve' in ref.lower() or 'CVE' in ref:
                        cve_list.append(ref.split('/')[-1])

            cve_id = cve_list[0] if cve_list else 'N/A'
            for c in cve_list:
                cves[c] += 1

            findings.append({
                'url': d.get('matched-at', d.get('host', '?')),
                'cve': cve_id,
                'severity': sev,
                'name': d.get('info',{}).get('name', '?'),
                'template': d.get('template-id', ''),
                'extracted': d.get('extracted-results', []),
            })
        except: pass

# Summary
icons = {'critical':'🔴','high':'🟠','medium':'🟡','low':'🔵','unknown':'⚪'}
print('# Mass Vulnerability Hunter Report')
print(f'Generated: {datetime.now().strftime(\"%Y-%m-%d %H:%M\")}')
print()
print('## Summary')
print('| Metric | Value |')
print('|--------|-------|')
print(f'| Total Findings | {len(findings)} |')
for sev in ['critical','high','medium','low']:
    if severities[sev]:
        print(f'| {icons.get(sev,\"?\")} {sev.capitalize()} | {severities[sev]} |')

if cves:
    print()
    print('## CVEs Detected')
    for cve, cnt in cves.most_common(30):
        print(f'- [{cve}](https://nvd.nist.gov/vuln/detail/{cve}): {cnt} occurrences')

print()
print('## All Findings')
for f in findings:
    icon = icons.get(f['severity'], '?')
    print(f'- {icon} [{f[\"cve\"]}] {f[\"name\"]} @ {f[\"url\"]}')
" > "$output" 2>/dev/null

  ok "Report: $output"
  cat "$output"
}

# ============================================================
# MAIN ENTRY POINT
# ============================================================

hunter_main() {
  local target_count="${1:-500}"
  local mode="${2:-auto}"  # auto|quick|deep

  echo ""
  echo "╔═══════════════════════════════════════════╗"
  echo "║     MASS VULNERABILITY HUNTER             ║"
  echo "╚═══════════════════════════════════════════╝"
  echo ""

  # Phase 1: Build target list
  log "PHASE 1: Target Discovery"
  discover_from_tranco "$target_count"
  discover_vulnerable_software
  discover_from_analyzer

  # Merge targets
  cat "$TARGETS_DIR"/hunt_*.txt "$WORK_DIR"/vuln_software_targets.txt 2>/dev/null | \
    sort -u > "$WORK_DIR/all_targets.txt"
  ok "Total unique targets: $(wc -l < "$WORK_DIR/all_targets.txt")"

  # Phase 2: Tech Detection + CVE Scan
  echo ""
  log "PHASE 2: Scanning"

  local tech_file="$WORK_DIR/tech_detected.json"
  detect_technologies "$WORK_DIR/all_targets.txt" "$tech_file"

  echo ""
  local findings_file=$(run_focused_cve_scan "$WORK_DIR/all_targets.txt" "$tech_file")

  # Phase 3: Analyze + Report
  echo ""
  log "PHASE 3: Analysis"
  analyze_findings "$findings_file"

  echo ""
  ok "Hunt complete!"
  echo "  Targets scanned: $(wc -l < "$WORK_DIR/all_targets.txt")"
  echo "  Findings: $(wc -l < "$findings_file" 2>/dev/null || echo 0)"
  echo "  Report: $REPORTS_DIR/hunter_report_*.md"
}

# Run if executed directly
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
  hunter_main "$@"
fi