commit 73d1640e25cacc9dc28b4345653caaa2addf23c6 Author: drjones Date: Tue Oct 6 23:43:38 2026 -0700 Snapshot: full project state diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24b2937 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +__pycache__/ +*.pyc +node_modules/ +.venv/ +venv/ +.env +*.db +*.sqlite* +*.log +.DS_Store +out/ +work/ +.pio/ +briefs/ +dns-backup/ +archive/ +*.png diff --git a/README.md b/README.md new file mode 100644 index 0000000..1c86ac3 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Threatmarket — Threat Intel Data Marketplace + +Public catalog + paid archive of REAL threat intelligence harvested by the +fleet. Metadata is public; payloads are paid — sats/BTC via BTCPay, download +as auto-zipped intel packs. + +## Pipeline +Fleet tools (OSINT Terminal's 454 tools, Lynx, Argus, research engine, market +crawls) continuously enumerate real data — subdomains, exposed services, cert +transparency, breach indicators, newly-registered domains. An auto-pipeline +(2×/day) packages a fresh intel pack from a random website/company into +structured JSON/CSV. Only real findings, never made-up data. + +`app.py` marketplace API · `cover_gen.py` pack covers · `SITE_SPEC.md` spec · +`deploy/` CT deployment · `archive/` generated packs (gitignored) + diff --git a/SITE_SPEC.md b/SITE_SPEC.md new file mode 100644 index 0000000..22c11e1 --- /dev/null +++ b/SITE_SPEC.md @@ -0,0 +1,25 @@ +# THREATMARKET — Threat Intel Data Marketplace + +Public catalog + paid archive of REAL threat intelligence harvested by the fleet. +Customers browse sample data, pay sats/BTC via BTCPay, download auto-zipped intel packs. + +## Business model +- Fleet tools (OSINT Terminal 454 tools, Lynx, Argus, research engine, market-intel crawls) + continuously enumerate REAL data: subdomains, exposed services, cert transparency, breach + indicators, newly-registered domains, market/lead datasets. +- Auto-pipeline (2x daily) packages a fresh intel pack from a RANDOM website/company + and archives it as a zip: /opt/threatmarket/archive/-.zip +- Each pack = structured JSON/CSV (not made-up data — only real findings from real scans) +- Site lists archive metadata (domain, date, finding counts, categories) WITHOUT the payload. + Buyer pays BTCPay -> webhook -> zip unlocked for browser download (one-time link). + +## Stack +- CT TBD "threatmarket" (Debian 12), Flask + nginx :80->app +- BTCPay store + webhook (payment confirmed -> unlock zip) +- Gitea repo, Umami beacon, BMAC footer, agent SEO (llms.txt) +- Cron: 2x daily random-target intel pack generation + +## Rules +- ZERO fabricated data. Every finding must come from a real scan run by real fleet tools. +- If a scan produces nothing, the pack says so. No filler, no example rows. +- Public pages show counts/metadata only; payloads only after payment. diff --git a/app.py b/app.py new file mode 100644 index 0000000..dd1e846 --- /dev/null +++ b/app.py @@ -0,0 +1,254 @@ +#!/opt/threatmarket/venv/bin/python3 +"""THREATMARKET — threat-intel marketplace. Metadata public, payloads paid.""" +import hashlib +import hmac +import json +import os +import secrets +import sqlite3 +import time + +import requests +from flask import Flask, Response, jsonify, redirect, render_template_string, request, send_file + +BASE = os.environ.get("TM_BASE", "/opt/threatmarket") +ARCHIVE = os.path.join(BASE, "archive") +MANIFEST = os.path.join(ARCHIVE, "manifest.json") +DB = os.path.join(BASE, "threatmarket.db") + +BTCPAY_URL = os.environ.get("BTCPAY_URL", "https://10.30.20.140").rstrip("/") +BTCPAY_STORE = os.environ["BTCPAY_STORE"] +BTCPAY_KEY = os.environ["BTCPAY_KEY"] +WEBHOOK_SECRET = os.environ["BTCPAY_WEBHOOK_SECRET"] +PACK_PRICE_USD = os.environ.get("TM_PRICE_USD", "12.00") + +app = Flask(__name__) + + +def db(): + con = sqlite3.connect(DB, timeout=10) + con.row_factory = sqlite3.Row + return con + + +def init_db(): + with db() as con: + con.execute( + """CREATE TABLE IF NOT EXISTS purchases ( + token TEXT PRIMARY KEY, pack_id TEXT, invoice_id TEXT UNIQUE, + status TEXT, created INTEGER, downloaded INTEGER DEFAULT 0)""" + ) + + +def packs(): + with open(MANIFEST) as f: + return json.load(f) + + +def get_pack(pack_id): + for p in packs(): + if p["pack_id"] == pack_id: + return p + return None + + +def file_sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + return h.hexdigest() + + +def btcpay_invoice(pack_id, token): + r = requests.post( + f"{BTCPAY_URL}/api/v1/stores/{BTCPAY_STORE}/invoices", + headers={"Authorization": f"token {BTCPAY_KEY}"}, + json={ + "amount": PACK_PRICE_USD, + "currency": "USD", + "metadata": {"orderId": token, "itemDesc": f"THREATMARKET intel pack {pack_id}"}, + "checkout": {"redirectURL": f"http://10.30.20.102/status/{token}", "redirectAutomatically": True}, + }, + verify=False, + timeout=30, + ) + r.raise_for_status() + return r.json() + + +# ---------- pages ---------- +BASE_CSS = """ +body{background:#0a0e14;color:#c9d1d9;font-family:'SF Mono',ui-monospace,Menlo,monospace;margin:0} +a{color:#58a6ff;text-decoration:none}a:hover{text-decoration:underline} +.wrap{max-width:960px;margin:0 auto;padding:40px 24px} +h1{color:#f0f6fc;letter-spacing:.18em}h1 .tm{color:#3fb950} +.tag{color:#8b949e;font-size:14px} +.card{background:#11161f;border:1px solid #1f2733;border-radius:10px;padding:18px 22px;margin:14px 0} +.card:hover{border-color:#3fb950} +.grid{display:flex;justify-content:space-between;align-items:center;gap:12px;flex-wrap:wrap} +.badge{background:#12261e;color:#3fb950;border:1px solid #1f4d33;border-radius:99px;padding:2px 10px;font-size:12px;margin-left:6px} +.count{color:#58a6ff;font-weight:bold} +.btn{display:inline-block;background:#238636;color:#fff;border-radius:8px;padding:10px 22px;font-weight:bold} +.btn:hover{text-decoration:none;background:#2ea043} +.price{color:#e3b341;font-size:20px;font-weight:bold} +footer{border-top:1px solid #1f2733;margin-top:48px;padding:22px 24px;text-align:center;color:#8b949e;font-size:13px} +.sha{color:#8b949e;font-size:11px;word-break:break-all} +""" + +FOOTER = """ +""" + + +def page(title, body): + return render_template_string( + "" + "{{t}}
{{body|safe}}
{{footer}}", + t=title, css=BASE_CSS, body=body, footer=FOOTER, + ) + + +@app.get("/") +def index(): + rows = "" + for p in packs(): + cats = "".join(f'{c}' for c in p["categories"]) + counts = " · ".join(f'{k}: {v}' for k, v in p["finding_counts"].items()) + rows += f"""
+
{p['domain']}{cats}
+ {p['date']} · {counts}
+ sha256 {p['sha256'][:32]}… · {p['size_bytes']//1024} KB
+
${PACK_PRICE_USD}
Buy pack
+
""" + body = f"""

THREATMARKET

+

Threat-intel packs harvested by a real scanning fleet. Certificate-transparency +logs, DNS liveness, exposed-service inventory. Metadata below is public; payloads unlock after BTC payment.

+{rows or '

No packs archived yet — the pipeline runs 2x daily.

'}""" + return page("THREATMARKET", body) + + +@app.get("/archive") +def archive(): + out = [ + {k: p[k] for k in ("pack_id", "domain", "date", "categories", "finding_counts", "size_bytes", "sha256")} + for p in packs() + ] + return jsonify(out) + + +@app.get("/buy/") +def buy_page(pack_id): + p = get_pack(pack_id) + if not p: + return page("404", "

Pack not found

"), 404 + counts = " · ".join(f"{k}: {v}" for k, v in p["finding_counts"].items()) + body = f"""

Pack: {p['domain']}

+
{p['date']} · {counts}
+sha256 {p['sha256']} · {p['size_bytes']//1024} KB zip
+

Price ${PACK_PRICE_USD} · paid in BTC over BTCPay. +After settlement you get a one-time download link for the pack zip.

+
""" + return page(f"Buy {pack_id}", body) + + +@app.post("/buy/") +def buy_go(pack_id): + p = get_pack(pack_id) + if not p: + return page("404", "

Pack not found

"), 404 + token = secrets.token_urlsafe(24) + inv = btcpay_invoice(pack_id, token) + with db() as con: + con.execute( + "INSERT INTO purchases (token, pack_id, invoice_id, status, created) VALUES (?,?,?,?,?)", + (token, pack_id, inv["id"], "pending", int(time.time())), + ) + return redirect(inv["checkoutLink"]) + + +@app.get("/status/") +def status(token): + with db() as con: + row = con.execute("SELECT * FROM purchases WHERE token=?", (token,)).fetchone() + if not row: + return page("404", "

Unknown order

"), 404 + if row["status"] == "paid" and not row["downloaded"]: + body = f"""

Payment confirmed ✅

Your one-time download is ready. +The link dies after the first successful download.

+Download {row['pack_id']}.zip""" + elif row["status"] == "paid": + body = "

Already downloaded

This one-time link has been consumed.

" + else: + body = f"""

Awaiting payment…

Invoice {row['invoice_id']} is +{row['status']}. This page refreshes every 15s.

+""" + return page("Order status", body) + + +@app.get("/download//") +def download(pack_id, token): + with db() as con: + row = con.execute("SELECT * FROM purchases WHERE token=?", (token,)).fetchone() + if not row or row["pack_id"] != pack_id: + return jsonify(error="invalid token"), 403 + if row["status"] != "paid": + return jsonify(error="payment not confirmed"), 402 + if row["downloaded"]: + return jsonify(error="one-time link already used"), 410 + con.execute("UPDATE purchases SET downloaded=1 WHERE token=?", (token,)) + p = get_pack(pack_id) + path = os.path.join(ARCHIVE, p["file"]) + return send_file(path, as_attachment=True, download_name=f"{pack_id}.zip") + + +@app.post("/webhook/btcpay") +def webhook(): + sig = request.headers.get("BTCPay-Sig", "") + expected = "sha256=" + hmac.new(WEBHOOK_SECRET.encode(), request.get_data(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(sig, expected): + return jsonify(error="bad signature"), 400 + data = request.get_json(force=True) + if data.get("type") == "InvoiceSettled": + inv = data["invoiceId"] + with db() as con: + con.execute("UPDATE purchases SET status='paid' WHERE invoice_id=? AND status='pending'", (inv,)) + return "", 200 + + +@app.get("/health") +def health(): + return jsonify(status="ok", service="threatmarket", packs=len(packs())) + + +@app.get("/robots.txt") +def robots(): + return Response("User-agent: *\nAllow: /\nDisallow: /download/\n", mimetype="text/plain") + + +@app.get("/llms.txt") +def llms(): + return Response( + "# THREATMARKET\n\n" + "Threat-intel marketplace selling archived reconnaissance packs (certificate-transparency " + "subdomain enumeration, DNS liveness, exposed-service inventory) collected by real fleet scans.\n\n" + "## Endpoints\n" + "- GET /: landing page, pack catalog with metadata only\n" + "- GET /archive: JSON list of packs (domain, date, categories, finding counts; no payloads)\n" + "- GET|POST /buy/{PACK_ID}: create a BTCPay invoice, pay in BTC\n" + "- GET /status/{TOKEN}: order status; one-time download link after settlement\n" + "- GET /download/{PACK_ID}/{TOKEN}: one-time zip download, requires webhook-confirmed payment\n" + "- GET /health: service health JSON\n\n" + "## Rules\n" + "- Public pages expose metadata/counts only; payloads require payment.\n" + "- Every finding comes from a real scan; empty scans ship as empty. Zero fabricated data.\n", + mimetype="text/plain", + ) + + +init_db() + +if __name__ == "__main__": + app.run(host="127.0.0.1", port=8500) diff --git a/cover_gen.py b/cover_gen.py new file mode 100644 index 0000000..00f2757 --- /dev/null +++ b/cover_gen.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Generate a cover image for a THREATMARKET intel pack via ComfyUI SDXL. + +Zero-failure design: tries ComfyUI (juggernautXL, then dreamshaperXL, then +sdxl-base), then falls back to a locally-rendered SVG->PNG-style placeholder +(a real, deterministic dark radar-brand cover - not fake data, just brand art). +Always writes /opt/threatmarket/static/covers/.png (or .svg) and +returns the web path, or None only if the filesystem itself is unwritable. + +Usage: from cover_gen import generate_cover + path = generate_cover("chewy.com", "2026-09-25") +""" +import base64 +import json +import os +import urllib.request + +COMFY = os.environ.get("COMFY_URL", "http://10.30.20.29:8188") +COVER_DIR = os.environ.get("COVER_DIR", "/opt/threatmarket/static/covers") +WEB_PREFIX = "/covers" +# Order = preference. Lightning models = few steps = fast + reliable. +CKPTS = ["flux1-schnell-fp8.safetensors", + "juggernautXL_ragnarok.safetensors", + "dreamshaperXL_lightningDPMSDE.safetensors"] +TIMEOUT_SUBMIT = 20 +TIMEOUT_POLL = 150 +TIMEOUT_FETCH = 30 + + +def _prompt_text(domain, date): + return ('dark hacker operations room at night, glowing green matrix rain code ' + 'walls, multiple monitors showing world-map network dashboards and ' + 'breach data, empty desk chair, dramatic cyan and green lighting, ' + 'cinematic depth of field, gritty cyberpunk atmosphere, wide dark ' + 'area at top for title text, professional threat-intelligence dossier ' + 'cover art, no text, no letters, no words, 4k') + + +def _workflow(ckpt, seed): + return { + "3": {"class_type": "KSampler", "inputs": { + "seed": seed, "steps": 20, "cfg": 1.8, "sampler_name": "euler", + "scheduler": "sgm_uniform", "denoise": 1.0, + "model": ["4", 0], "positive": ["6", 0], "negative": ["7", 0], + "latent_image": ["5", 0]}}, + "4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": ckpt}}, + "5": {"class_type": "EmptyLatentImage", "inputs": {"width": 1024, "height": 640, "batch_size": 1}}, + "6": {"class_type": "CLIPTextEncode", "inputs": {"text": "placeholder", "clip": ["4", 1]}}, + "7": {"class_type": "CLIPTextEncode", "inputs": {"text": "text, watermark, blurry, low quality, oversaturated", "clip": ["4", 1]}}, + "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["4", 2]}}, + "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "tmcover", "images": ["8", 0]}}, + } + + +def _comfy_generate(domain, date, seed): + """Try each checkpoint; return PNG bytes or None.""" + for ckpt in CKPTS: + try: + wf = _workflow(ckpt, seed) + # fix node 6 text properly (dict-literal scoping hack above) + wf["6"]["inputs"]["text"] = _prompt_text(domain, date) + req = urllib.request.Request( + COMFY + "/prompt", + data=json.dumps({"prompt": wf}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=TIMEOUT_SUBMIT) as r: + resp = json.loads(r.read()) + pid = resp.get("prompt_id") + if not pid: + continue + import time + deadline = time.time() + TIMEOUT_POLL + while time.time() < deadline: + with urllib.request.urlopen(COMFY + "/history/" + pid, timeout=15) as r: + hist = json.loads(r.read()) + if pid in hist and hist[pid].get("outputs"): + outs = hist[pid]["outputs"] + for node in outs.values(): + for img in node.get("images", []): + fn = img["filename"] + sub = img.get("subfolder", "") + imgtype = img.get("type", "output") + url = f"{COMFY}/view?filename={fn}&subfolder={sub}&type={imgtype}" + with urllib.request.urlopen(url, timeout=TIMEOUT_FETCH) as fr: + return fr.read() + time.sleep(3) + except Exception: + continue + return None + + +def _fallback_svg(domain, date): + """Deterministic branded SVG cover - always succeeds if fs writable.""" + seed = sum(ord(c) for c in domain) % 360 + svg = f""" + + + +""" + for i in range(6): + svg += f"" + svg += f"" + import math + for i in range(14): + x = 120 + (i * 61) % 700 + y = 420 + (i * 37) % 160 + r = 2 + (i % 3) + svg += f"" + svg += f""" +INTEL PACK +{domain} +{date} — THREATMARKET +""" + return svg.encode() + + +def generate_cover(domain, date): + """Returns web path of cover image, or None. Never raises.""" + os.makedirs(COVER_DIR, exist_ok=True) + pid = (domain + "-" + date.replace("-", "")).replace("/", "_") + png_path = os.path.join(COVER_DIR, pid + ".png") + svg_path = os.path.join(COVER_DIR, pid + ".svg") + if os.path.exists(png_path): + return f"{WEB_PREFIX}/{pid}.png" + if os.path.exists(svg_path): + return f"{WEB_PREFIX}/{pid}.svg" + seed = sum(ord(c) for c in pid) % (2 ** 31) + png = _comfy_generate(domain, date, seed) + if png: + try: + with open(png_path, "wb") as f: + f.write(png) + try: + from text_stamp import stamp + stamp(png_path, domain) + except Exception: + pass # art still fine even if stamp fails + return f"{WEB_PREFIX}/{pid}.png" + except OSError: + pass + try: + with open(svg_path, "wb") as f: + f.write(_fallback_svg(domain, date)) + return f"{WEB_PREFIX}/{pid}.svg" + except OSError: + return None + + +if __name__ == "__main__": + import sys + d = sys.argv[1] if len(sys.argv) > 1 else "mozilla.org" + dt = sys.argv[2] if len(sys.argv) > 2 else "2026-09-24" + print(generate_cover(d, dt) or "FAILED") diff --git a/deploy/env b/deploy/env new file mode 100644 index 0000000..0ccd736 --- /dev/null +++ b/deploy/env @@ -0,0 +1,5 @@ +BTCPAY_URL=https://10.30.20.140 +BTCPAY_STORE=GUZqSmsabLYRQ24zRMGAmepTXxdvFagenCBBCQNr1773 +BTCPAY_KEY=385204347c761fa8bac2119f8fbc339b033ea490 +BTCPAY_WEBHOOK_SECRET=d68ccd12547de8253a74be0e5c80927d075360e99402342a +TM_PRICE_USD=12.00 diff --git a/deploy/nginx-threatmarket.conf b/deploy/nginx-threatmarket.conf new file mode 100644 index 0000000..fe30105 --- /dev/null +++ b/deploy/nginx-threatmarket.conf @@ -0,0 +1,13 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location / { + proxy_pass http://127.0.0.1:8500; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/deploy/threatmarket.service b/deploy/threatmarket.service new file mode 100644 index 0000000..6157c89 --- /dev/null +++ b/deploy/threatmarket.service @@ -0,0 +1,14 @@ +[Unit] +Description=THREATMARKET threat-intel marketplace (gunicorn) +After=network.target + +[Service] +Type=simple +WorkingDirectory=/opt/threatmarket +EnvironmentFile=/opt/threatmarket/env +ExecStart=/opt/threatmarket/venv/bin/gunicorn --workers 2 --bind 127.0.0.1:8500 app:app +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/manifest_patch.py b/manifest_patch.py new file mode 100644 index 0000000..a9cf468 --- /dev/null +++ b/manifest_patch.py @@ -0,0 +1,24 @@ +import json, hashlib, os + +MAN = "/opt/threatmarket/archive/manifest.json" + + +def update_manifest(zip_path, domain, date, categories, finding_count): + """Register a finished pack in the archive manifest so the web app lists it.""" + h = hashlib.sha256(open(zip_path, "rb").read()).hexdigest() + man = json.load(open(MAN)) if os.path.exists(MAN) else [] + pid = domain + "-" + date.replace("-", "") + if not any(m.get("pack_id") == pid for m in man): + man.append({ + "pack_id": pid, + "domain": domain, + "date": date, + "categories": categories, + "finding_counts": {"total_findings": finding_count}, + "size_bytes": os.path.getsize(zip_path), + "sha256": h, + "file": os.path.basename(zip_path), + }) + tmp = MAN + ".tmp" + json.dump(man, open(tmp, "w"), indent=1) + os.replace(tmp, MAN)