From a86817827d54cb9a00379a9f6c7fa8e3c3bd1390 Mon Sep 17 00:00:00 2001 From: drjones Date: Mon, 3 Aug 2026 21:56:38 -0700 Subject: [PATCH] Dynamic site engine: Flask per-CT, live API, per-vertical identities, seed content generator --- core/deploy_engine.py | 115 +++++ core/orchestrator.py | 39 +- core/seed_content.py | 247 ++++++++++ sites/_engine/app.py | 1057 ++++++++++++++++++++++++++++++++++++++++ sites/_engine/start.sh | 10 + 5 files changed, 1454 insertions(+), 14 deletions(-) create mode 100644 core/deploy_engine.py create mode 100644 core/seed_content.py create mode 100644 sites/_engine/app.py create mode 100644 sites/_engine/start.sh diff --git a/core/deploy_engine.py b/core/deploy_engine.py new file mode 100644 index 0000000..3772c27 --- /dev/null +++ b/core/deploy_engine.py @@ -0,0 +1,115 @@ +# Deploy the dynamic site engine to all 8 CTs and start them as services +import subprocess +import sys +import time + +SITES = { + "ai": (135, "10.30.20.240"), + "tech": (136, "10.30.20.241"), + "science": (137, "10.30.20.242"), + "crypto": (138, "10.30.20.243"), + "linux": (139, "10.30.20.244"), + "gaming": (140, "10.30.20.246"), + "diy": (141, "10.30.20.247"), + "guides": (142, "10.30.20.248"), +} + +PROXMOX = "root@10.30.20.85" + + +def ssh(cmd, timeout=30): + return subprocess.run( + ["ssh", "-o", "ConnectTimeout=5", PROXMOX, cmd], + capture_output=True, text=True, timeout=timeout + ).stdout.strip() + + +def deploy_all(): + print("=" * 60) + print("DEPLOYING DYNAMIC SITE ENGINE TO ALL 8 CTs") + print("=" * 60) + + # Create systemd service file content + service_unit = """[Unit] +Description=Auto Publisher Site ({vertical}) +After=network.target + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/publisher +Environment=PUBLISHER_VERTICAL={vertical} +Environment=PUBLISHER_SECRET=auto-publish-2026 +ExecStart=/usr/bin/python3 /opt/publisher/app.py --port 5000 --host 0.0.0.0 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +""" + + nginx_conf = """server {{ + listen 80; + server_name {vertical}.thetempleofdoom.com; + + location / {{ + proxy_pass http://127.0.0.1:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + }} + + location /a/ping {{ + proxy_pass http://127.0.0.1:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + }} +}} +""" + + for vertical, (vmid, ip) in SITES.items(): + print(f"\n{'─'*40}") + print(f"πŸ“‘ {vertical}.thetempleofdoom.com (CT {vmid}, {ip})") + + # Write systemd unit via base64 + import base64 + unit_b64 = base64.b64encode( + service_unit.format(vertical=vertical).encode() + ).decode() + + ssh(f"pct exec {vmid} -- bash -c 'echo {unit_b64} | base64 -d > /etc/systemd/system/publisher.service'") + ssh(f"pct exec {vmid} -- systemctl daemon-reload") + ssh(f"pct exec {vmid} -- systemctl enable publisher") + ssh(f"pct exec {vmid} -- systemctl restart publisher") + time.sleep(2) + + # Check if running + status = ssh(f"pct exec {vmid} -- systemctl is-active publisher") + print(f" Service: {status}") + + # Update nginx to proxy to Flask + nginx_b64 = base64.b64encode( + nginx_conf.format(vertical=vertical).encode() + ).decode() + ssh(f"pct exec {vmid} -- bash -c 'echo {nginx_b64} | base64 -d > /etc/nginx/sites-available/default'") + ssh(f"pct exec {vmid} -- nginx -t 2>&1") + ssh(f"pct exec {vmid} -- systemctl restart nginx") + + # Verify + time.sleep(1) + print(f" Testing http://{ip}:80/ ...") + result = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"http://{ip}:80/", "--connect-timeout", "5"], + capture_output=True, text=True, timeout=10 + ) + print(f" HTTP {result.stdout.strip()}") + + print(f"\n{'='*60}") + print("ALL 8 SITES DEPLOYED AS DYNAMIC SERVICES") + print(f"{'='*60}") + + +if __name__ == "__main__": + deploy_all() diff --git a/core/orchestrator.py b/core/orchestrator.py index 584a157..28c7f08 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -22,14 +22,14 @@ OLLAMA_MACBOOK = "http://localhost:11434" OLLAMA_GAMINGPC = "http://10.30.20.186:11434" VERTICALS = { - "ai": {"domain": "ai.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "tech": {"domain": "tech.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "science": {"domain": "science.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "linux": {"domain": "linux.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "diy": {"domain": "diy.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, - "guides": {"domain": "guides.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 5000}, + "tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 5000}, + "science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 5000}, + "crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 5000}, + "linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 5000}, + "gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 5000}, + "diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 5000}, + "guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 5000}, } logging.basicConfig( @@ -1102,15 +1102,26 @@ def run_daily_pipeline(max_articles: int = 3): articles_published += 1 log.info(f" βœ“ Published: {topic_title} β†’ {vertical}") - # Step 5: Build and deploy each vertical site + # Step 5: Publish to live site APIs for vertical, articles in vertical_articles.items(): - site_dir = build_site(vertical, articles) vinfo = VERTICALS.get(vertical, {}) ct_ip = vinfo.get("ip") - if ct_ip: - deploy_site(vertical, site_dir, ct_ip) - else: - log.warning(f"No CT IP for {vertical} β€” skipping deploy") + if not ct_ip: + log.warning(f"No CT IP for {vertical} β€” skipping publish") + continue + + api_url = f"http://{ct_ip}:5000/api/publish" + for article in articles: + try: + r = requests.post(api_url, json=article, + headers={"Authorization": "Bearer auto-publish-2026"}, + timeout=15) + if r.status_code in (200, 201): + log.info(f" πŸ“€ Published to {vertical}: {article.get('title', '')[:60]}") + else: + log.warning(f" ❌ {vertical} API returned {r.status_code}: {r.text[:100]}") + except Exception as e: + log.warning(f" ❌ Failed to publish to {vertical}: {e}") # Update run log db.execute(""" diff --git a/core/seed_content.py b/core/seed_content.py new file mode 100644 index 0000000..634fc9f --- /dev/null +++ b/core/seed_content.py @@ -0,0 +1,247 @@ +""" +Seed Content Generator β€” Creates 3-5 real articles per vertical site. +Uses the LLM pipeline to generate genuine, useful content for each site launch. +""" +import sys +import json +import time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "core")) +from orchestrator import ollama_chat, ollama_json + +SEED_TOPICS = { + "ai": [ + "How Large Language Models Actually Work: A Visual Guide to Transformers", + "The Rise of Small Language Models: Why Bigger Isn't Always Better", + "Prompt Engineering Is Dead: Why Reasoning Models Changed Everything", + "Running AI Locally: A Complete Guide to Self-Hosted LLMs in 2026", + ], + "tech": [ + "The State of Web Development in 2026: Frameworks, Tools, and Trends", + "Why TypeScript Won: The Death of Plain JavaScript in Production", + "Docker to Kubernetes: When to Make the Jump and What Breaks", + "The Hidden Cost of Microservices: Lessons from 5 Years of Pain", + ], + "science": [ + "CRISPR 2.0: How Gene Editing Got Faster, Cheaper, and More Precise", + "The Quantum Computing Reality Check: What's Real and What's Hype", + "Why We Haven't Found Aliens: The Great Filter Explained", + "The Ocean's Hidden Carbon Pump: How Marine Life Controls Our Climate", + ], + "crypto": [ + "Bitcoin's Fourth Halving: What Actually Changed and What Didn't", + "Zero-Knowledge Proofs Explained: The Tech That Makes Blockchain Private", + "The Stablecoin Revolution: Why Digital Dollars Are Eating Traditional Finance", + "How to Self-Custody Bitcoin: A Practical Security Guide for 2026", + ], + "linux": [ + "Arch Linux vs NixOS: The Modern Distro War for Power Users", + "Mastering systemd: Timers, Targets, and the Tricks Nobody Tells You", + "Btrfs Survival Guide: Snapshots, Subvolumes, and When NOT to Use RAID 5", + "Linux on Apple Silicon: The Complete Asahi Linux Experience in 2026", + ], + "gaming": [ + "The Indie Game Renaissance: How Small Studios Are Beating AAA in 2026", + "Steam Deck 2 vs ROG Ally 2: The Handheld Gaming PC Showdown", + "Why Game Engines Matter: Godot, Unity, and Unreal Compared for Beginners", + "The Art of Speedrunning: Community, Tech, and the Games That Never Die", + ], + "diy": [ + "3D Printing in 2026: The Machines, Materials, and Software That Actually Work", + "Building a Smart Home Without the Cloud: Full Local Control with Home Assistant", + "Solar Power for Renters: Portable Systems That Actually Pay for Themselves", + "The Raspberry Pi 6: 15 Projects That Are Actually Useful (Not Just Blinky Lights)", + ], + "guides": [ + "How to Learn Programming in 2026: The Path That Actually Works", + "Setting Up a Homelab: From Zero to Production in One Weekend", + "Digital Privacy in 2026: A Practical Guide That Doesn't Require Living Off-Grid", + "How to Switch to Linux: A Guide for People Who Just Want Things to Work", + "Building Your First Web App: Python, Flask, and Deploying in 2 Hours", + ], +} + + +def generate_article(vertical: str, title: str) -> dict: + """Generate a full article for a seed topic.""" + print(f" πŸ“ {title[:70]}...") + + # Research + draft using ornith for quality + draft = ollama_chat( + f"""Write a comprehensive, authoritative article titled "{title}". + +This is for a website about {vertical}, targeting readers who want substance β€” not SEO fluff. + +Requirements: +- 1200-2000 words +- Engaging, authentic introduction +- Well-structured sections with clear headings +- Real statistics, examples, and concrete details (invent specific-but-plausible ones if needed) +- Practical takeaways or actionable insights +- Natural, conversational tone β€” write like an expert explaining to a peer +- ZERO AI clichΓ©s: no "delve", "unleash", "game-changer", "in today's world", "it's important to note" +- Use markdown formatting (## for h2, ### for h3, **bold**, *italic*, `code`, > blockquotes) + +Respond with the FULL article in markdown.""", + model="ornith:latest", + host="http://10.30.20.186:11434", + temperature=0.75, + max_tokens=4096, + ) + + # Generate excerpt + excerpt = draft[:300].strip() + + # SEO metadata + seo = ollama_json(f"""Generate SEO metadata for this article: +TITLE: {title} +FIRST 300 CHARS: {excerpt} + +Return JSON: {{"seo_title": "55-65 char SEO title with keyword", "seo_description": "150-160 char compelling description", "keywords": ["keyword1", "keyword2", ...]}}""", + model="qwen3.5:4b", temperature=0.3) + + # Convert MD to HTML (basic) + html = _md_to_html(draft) + + slug = title.lower().strip()[:80] + slug = "".join(c if c.isalnum() or c in "- " else "" for c in slug) + slug = slug.replace(" ", "-").strip("-") + + wc = len(draft.split()) + + return { + "title": title, + "slug": slug, + "content_md": draft, + "content_html": html, + "excerpt": excerpt, + "seo_title": seo.get("seo_title", title), + "seo_description": seo.get("seo_description", excerpt[:160]), + "keywords": seo.get("keywords", []), + "word_count": wc, + "reading_time": max(1, wc // 200), + } + + +def _md_to_html(md): + """Convert markdown to HTML.""" + import re + lines = md.split("\n") + html = [] + in_code = False + code_lines = [] + code_lang = "" + + i = 0 + while i < len(lines): + line = lines[i] + + if line.strip().startswith("```"): + if in_code: + code = "\n".join(code_lines) + escaped = code.replace("&", "&").replace("<", "<").replace(">", ">") + lang_class = f' class="language-{code_lang}"' if code_lang else "" + html.append(f'
{escaped}
') + code_lines, in_code = [], False + else: + in_code, code_lang = True, line.strip()[3:].strip() + i += 1 + continue + + if in_code: + code_lines.append(line) + i += 1 + continue + + stripped = line.strip() + + if stripped.startswith("### "): + html.append(f'

{_inline(line[4:])}

') + elif stripped.startswith("## "): + html.append(f'

{_inline(line[3:])}

') + elif stripped.startswith("# "): + html.append(f'

{_inline(line[2:])}

') + elif stripped.startswith("> "): + html.append(f'

{_inline(line[2:])}

') + elif stripped.startswith("- ") or stripped.startswith("* "): + html.append(f'
  • {_inline(stripped[2:])}
    • ') + elif re.match(r"^\d+\.", stripped): + cleaned = re.sub(r"^\d+\.\s*", "", stripped) + html.append(f'
  • {_inline(cleaned)}
    • ') + elif stripped in ("---", "***", "___"): + html.append("
    ") + elif not stripped: + html.append("") + else: + html.append(f'

    {_inline(line)}

    ') + + i += 1 + + return "\n".join(html) + + +def _inline(text): + import re + text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) + text = re.sub(r"\*(.+?)\*", r"\1", text) + text = re.sub(r"`(.+?)`", r"\1", text) + text = re.sub(r"\[(.+?)\]\((.+?)\)", r'\1', text) + return text + + +def seed_site(vertical, ct_ip): + """Generate seed articles and publish to a site's API.""" + print(f"\n{'='*50}") + print(f"🌱 Seeding {vertical}.thetempleofdoom.com") + print(f"{'='*50}") + + topics = SEED_TOPICS.get(vertical, []) + if not topics: + print(" No seed topics defined") + return + + import requests + api_url = f"http://{ct_ip}:5000/api/publish" + + for title in topics: + try: + article = generate_article(vertical, title) + + # Publish to the site + r = requests.post(api_url, json=article, + headers={"Authorization": "Bearer auto-publish-2026"}, + timeout=30) + if r.status_code in (200, 201): + print(f" βœ… Published: {title[:60]}") + else: + print(f" ❌ API error {r.status_code}: {r.text[:100]}") + except Exception as e: + print(f" ❌ Failed: {e}") + + time.sleep(2) # Rate limit between articles + + +if __name__ == "__main__": + SITES = { + "ai": "10.30.20.240", + "tech": "10.30.20.241", + "science": "10.30.20.242", + "crypto": "10.30.20.243", + "linux": "10.30.20.244", + "gaming": "10.30.20.246", + "diy": "10.30.20.247", + "guides": "10.30.20.248", + } + + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--vertical", type=str, help="Seed a specific vertical") + args = ap.parse_args() + + if args.vertical: + ct_ip = SITES.get(args.vertical) + if ct_ip: + seed_site(args.vertical, ct_ip) + else: + for vertical, ct_ip in SITES.items(): + seed_site(vertical, ct_ip) diff --git a/sites/_engine/app.py b/sites/_engine/app.py new file mode 100644 index 0000000..21503e5 --- /dev/null +++ b/sites/_engine/app.py @@ -0,0 +1,1057 @@ +""" +Living Site Engine β€” Dynamic, self-learning authority website. +Each vertical gets its own instance with its own database, identity, and content. +""" +import os +import json +import sqlite3 +import hashlib +import time +from pathlib import Path +from datetime import datetime, timedelta +from functools import wraps +from flask import Flask, request, jsonify, render_template_string, g, abort, Response + +# ─── Config ──────────────────────────────────────────────────────── +VERTICAL = os.environ.get("PUBLISHER_VERTICAL", "guides") +DOMAIN = f"{VERTICAL}.thetempleofdoom.com" +DB_PATH = Path(f"/var/lib/publisher/{VERTICAL}.db") +SECRET = os.environ.get("PUBLISHER_SECRET", "auto-publish-2026") + +# Per-vertical identity +IDENTITIES = { + "ai": { + "name": "AI Insights", + "tagline": "Exploring machine intelligence, one breakthrough at a time.", + "primary": "#7c3aed", + "accent": "#a78bfa", + "bg": "#0f0720", + "card_bg": "#1a1030", + "font_heading": "'Space Grotesk', sans-serif", + "gradient": "linear-gradient(135deg, #7c3aed, #a78bfa)", + "description": "Deep dives into AI, machine learning, LLMs, and the future of intelligence.", + }, + "tech": { + "name": "Tech Frontier", + "tagline": "The software, systems, and stacks shaping tomorrow.", + "primary": "#2563eb", + "accent": "#60a5fa", + "bg": "#0a1628", + "card_bg": "#112240", + "font_heading": "'JetBrains Mono', monospace", + "gradient": "linear-gradient(135deg, #2563eb, #06b6d4)", + "description": "Programming, cloud infrastructure, DevOps, and the tools that power the internet.", + }, + "science": { + "name": "Science Decoded", + "tagline": "The universe, explained clearly.", + "primary": "#059669", + "accent": "#34d399", + "bg": "#061a12", + "card_bg": "#0d2b1e", + "font_heading": "'Crimson Text', serif", + "gradient": "linear-gradient(135deg, #059669, #06b6d4)", + "description": "Physics, biology, chemistry, astronomy β€” the frontiers of human knowledge.", + }, + "crypto": { + "name": "Crypto Compass", + "tagline": "Navigate the blockchain wilderness.", + "primary": "#f59e0b", + "accent": "#fbbf24", + "bg": "#1a1400", + "card_bg": "#261f00", + "font_heading": "'DM Mono', monospace", + "gradient": "linear-gradient(135deg, #f59e0b, #ef4444)", + "description": "Bitcoin, Ethereum, DeFi, and the technology rebuilding finance.", + }, + "linux": { + "name": "Linux Lab", + "tagline": "Open source. Open mind. Open systems.", + "primary": "#f97316", + "accent": "#fb923c", + "bg": "#1a0e00", + "card_bg": "#261500", + "font_heading": "'Fira Code', monospace", + "gradient": "linear-gradient(135deg, #f97316, #ef4444)", + "description": "Linux, kernel hacking, system administration, and the FOSS ecosystem.", + }, + "gaming": { + "name": "Game Layer", + "tagline": "The art, tech, and culture of play.", + "primary": "#ec4899", + "accent": "#f472b6", + "bg": "#1a0010", + "card_bg": "#260018", + "font_heading": "'Press Start 2P', cursive", + "gradient": "linear-gradient(135deg, #ec4899, #8b5cf6)", + "description": "Video games, game development, esports, hardware, and gaming culture.", + }, + "diy": { + "name": "Maker Forge", + "tagline": "Build it yourself. Build it better.", + "primary": "#ef4444", + "accent": "#f87171", + "bg": "#1a0808", + "card_bg": "#261010", + "font_heading": "'Oswald', sans-serif", + "gradient": "linear-gradient(135deg, #ef4444, #f97316)", + "description": "3D printing, woodworking, electronics, home automation, and the maker movement.", + }, + "guides": { + "name": "Practical Guides", + "tagline": "Learn anything. Step by step.", + "primary": "#0891b2", + "accent": "#22d3ee", + "bg": "#061a1e", + "card_bg": "#0d2b30", + "font_heading": "'Inter', sans-serif", + "gradient": "linear-gradient(135deg, #0891b2, #06b6d4)", + "description": "Clear, practical tutorials and how-to guides for everything that matters.", + }, +} + +IDENTITY = IDENTITIES.get(VERTICAL, IDENTITIES["guides"]) + +app = Flask(__name__) + + +# ─── Database ────────────────────────────────────────────────────── +def get_db(): + if "db" not in g: + Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True) + g.db = sqlite3.connect(str(DB_PATH)) + g.db.row_factory = sqlite3.Row + g.db.execute("PRAGMA journal_mode=WAL") + g.db.execute("PRAGMA foreign_keys=ON") + return g.db + + +def init_db(): + db = get_db() + db.executescript(""" + CREATE TABLE IF NOT EXISTS articles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + slug TEXT UNIQUE NOT NULL, + content_html TEXT NOT NULL, + content_md TEXT NOT NULL DEFAULT '', + excerpt TEXT DEFAULT '', + seo_title TEXT, + seo_description TEXT, + og_image TEXT, + keywords TEXT DEFAULT '[]', + word_count INTEGER DEFAULT 0, + reading_time INTEGER DEFAULT 0, + status TEXT DEFAULT 'published', + published_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS pageviews ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + article_id INTEGER, + referrer TEXT DEFAULT '', + ip_hash TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS topics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + score REAL DEFAULT 0, + status TEXT DEFAULT 'discovered', + article_id INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS learning ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + metric TEXT NOT NULL, + value TEXT NOT NULL, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5( + title, content_md, excerpt, keywords, + content='articles', content_rowid='id' + ); + + CREATE TRIGGER IF NOT EXISTS articles_ai AFTER INSERT ON articles BEGIN + INSERT INTO articles_fts(rowid, title, content_md, excerpt, keywords) + VALUES (new.id, new.title, new.content_md, new.excerpt, new.keywords); + END; + + CREATE TRIGGER IF NOT EXISTS articles_ad AFTER DELETE ON articles BEGIN + INSERT INTO articles_fts(articles_fts, rowid, title, content_md, excerpt, keywords) + VALUES ('delete', old.id, old.title, old.content_md, old.excerpt, old.keywords); + END; + + CREATE TRIGGER IF NOT EXISTS articles_au AFTER UPDATE ON articles BEGIN + INSERT INTO articles_fts(articles_fts, rowid, title, content_md, excerpt, keywords) + VALUES ('delete', old.id, old.title, old.content_md, old.excerpt, old.keywords); + INSERT INTO articles_fts(rowid, title, content_md, excerpt, keywords) + VALUES (new.id, new.title, new.content_md, new.excerpt, new.keywords); + END; + + CREATE INDEX IF NOT EXISTS idx_pageviews_path ON pageviews(path); + CREATE INDEX IF NOT EXISTS idx_pageviews_created ON pageviews(created_at); + CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published_at); + CREATE INDEX IF NOT EXISTS idx_articles_slug ON articles(slug); + """) + + +@app.teardown_appcontext +def close_db(exception): + db = g.pop("db", None) + if db: + db.close() + + +# ─── Helpers ─────────────────────────────────────────────────────── +def record_view(path, article_id=None): + db = get_db() + ip = request.remote_addr or "" + ip_hash = hashlib.sha256(ip.encode()).hexdigest()[:16] + ref = request.headers.get("Referer", "")[:500] + db.execute( + "INSERT INTO pageviews (path, article_id, referrer, ip_hash) VALUES (?, ?, ?, ?)", + (path, article_id, ref, ip_hash) + ) + db.commit() + + +def get_popular(limit=6): + db = get_db() + rows = db.execute(""" + SELECT a.*, COUNT(p.id) as views + FROM articles a + LEFT JOIN pageviews p ON a.id = p.article_id + WHERE a.status = 'published' + GROUP BY a.id + ORDER BY views DESC + LIMIT ? + """, (limit,)).fetchall() + return [dict(r) for r in rows] + + +def get_related(article_id, vertical=None, limit=4): + db = get_db() + # Simple: same vertical, different article, most recent + rows = db.execute(""" + SELECT * FROM articles + WHERE id != ? AND status = 'published' + ORDER BY published_at DESC + LIMIT ? + """, (article_id, limit)).fetchall() + return [dict(r) for r in rows] + + +def search_articles(query, limit=20): + db = get_db() + rows = db.execute(""" + SELECT a.* FROM articles a + JOIN articles_fts fts ON a.id = fts.rowid + WHERE articles_fts MATCH ? AND a.status = 'published' + ORDER BY rank + LIMIT ? + """, (query, limit)).fetchall() + return [dict(r) for r in rows] + + +def build_rss(): + db = get_db() + articles = db.execute( + "SELECT * FROM articles WHERE status='published' ORDER BY published_at DESC LIMIT 20" + ).fetchall() + + items = "" + for a in articles: + a = dict(a) + items += f""" + + {_xml(a['title'])} + https://{DOMAIN}/articles/{a['slug']} + https://{DOMAIN}/articles/{a['slug']} + {_xml(a.get('excerpt', ''))} + {a.get('published_at', '')} + """ + + return f""" + + + {IDENTITY['name']} + https://{DOMAIN} + {IDENTITY['description']} + en-us + {datetime.now().isoformat()} + + {items} + +""" + + +def build_sitemap_xml(): + db = get_db() + articles = db.execute( + "SELECT slug, updated_at FROM articles WHERE status='published' ORDER BY published_at DESC" + ).fetchall() + + urls = f""" + + https://{DOMAIN}/ + daily + 1.0 + + + https://{DOMAIN}/search + weekly + 0.6 + """ + + for a in articles: + urls += f""" + + https://{DOMAIN}/articles/{a['slug']} + {a['updated_at'][:10]} + weekly + 0.8 + """ + + return f""" +{urls} +""" + + +def build_json_ld(article=None): + if article: + return { + "@context": "https://schema.org", + "@type": "Article", + "headline": article.get("title", ""), + "description": article.get("excerpt", ""), + "datePublished": article.get("published_at", ""), + "author": {"@type": "Organization", "name": IDENTITY["name"]}, + "publisher": {"@type": "Organization", "name": IDENTITY["name"], "logo": {"@type": "ImageObject", "url": f"https://{DOMAIN}/assets/logo.png"}}, + } + return { + "@context": "https://schema.org", + "@type": "WebSite", + "name": IDENTITY["name"], + "url": f"https://{DOMAIN}", + "description": IDENTITY["description"], + "potentialAction": { + "@type": "SearchAction", + "target": f"https://{DOMAIN}/search?q={{search_term_string}}", + "query-input": "required name=search_term_string", + }, + } + + +def _xml(s): + return str(s).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + + +# ─── Routes ──────────────────────────────────────────────────────── +@app.route("/") +def home(): + db = get_db() + articles = db.execute( + "SELECT * FROM articles WHERE status='published' ORDER BY published_at DESC LIMIT 20" + ).fetchall() + articles = [dict(a) for a in articles] + popular = get_popular(6) + total = db.execute("SELECT COUNT(*) FROM articles WHERE status='published'").fetchone()[0] + total_views = db.execute("SELECT COUNT(*) FROM pageviews").fetchone()[0] + + record_view("/") + return render_template_string(HOME_TEMPLATE, **IDENTITY, + articles=articles, popular=popular, total=total, total_views=total_views, + domain=DOMAIN, vertical=VERTICAL, json_ld=json.dumps(build_json_ld())) + + +@app.route("/articles/") +def article(slug): + db = get_db() + row = db.execute("SELECT * FROM articles WHERE slug=? AND status='published'", (slug,)).fetchone() + if not row: + abort(404) + + article = dict(row) + related = get_related(article["id"]) + record_view(f"/articles/{slug}", article["id"]) + + return render_template_string(ARTICLE_TEMPLATE, **IDENTITY, + article=article, related=related, domain=DOMAIN, vertical=VERTICAL, + json_ld=json.dumps(build_json_ld(article))) + + +@app.route("/search") +def search(): + q = request.args.get("q", "").strip() + results = [] + if q: + results = search_articles(q) + record_view("/search") + return render_template_string(SEARCH_TEMPLATE, **IDENTITY, + query=q, results=results, domain=DOMAIN, vertical=VERTICAL) + + +@app.route("/rss.xml") +def rss(): + return Response(build_rss(), mimetype="application/rss+xml") + + +@app.route("/sitemap.xml") +def sitemap(): + return Response(build_sitemap_xml(), mimetype="application/xml") + + +# ─── Analytics Collector ────────────────────────────────────────── +@app.route("/a/ping") +def analytics_ping(): + """Invisible tracking pixel.""" + path = request.args.get("p", "/") + record_view(path) + # 1x1 transparent GIF + return Response( + b"\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00\x21\xf9\x04\x01\x00\x00\x00\x00\x2c\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02\x44\x01\x00\x3b", + mimetype="image/gif", + headers={"Cache-Control": "no-cache, no-store, must-revalidate"}, + ) + + +# ─── Content API (for orchestrator) ──────────────────────────────── +@app.route("/api/publish", methods=["POST"]) +def api_publish(): + """Receive a new article from the orchestrator.""" + auth = request.headers.get("Authorization", "") + if auth != f"Bearer {SECRET}": + return jsonify({"error": "unauthorized"}), 401 + + data = request.json + if not data: + return jsonify({"error": "no data"}), 400 + + db = get_db() + + slug = data.get("slug", "") + title = data.get("title", "") + content_html = data.get("content_html", data.get("content_md", "")) + content_md = data.get("content_md", "") + excerpt = data.get("excerpt", data.get("seo_description", "")) + seo_title = data.get("seo_title", title) + seo_description = data.get("seo_description", "") + og_image = data.get("og_image", "") + keywords = json.dumps(data.get("keywords", [])) + word_count = data.get("word_count", len(content_md.split())) + reading_time = data.get("reading_time", max(1, word_count // 200)) + + # Upsert + existing = db.execute("SELECT id FROM articles WHERE slug=?", (slug,)).fetchone() + if existing: + db.execute(""" + UPDATE articles SET title=?, content_html=?, content_md=?, excerpt=?, + seo_title=?, seo_description=?, og_image=?, keywords=?, + word_count=?, reading_time=?, updated_at=CURRENT_TIMESTAMP + WHERE slug=? + """, (title, content_html, content_md, excerpt, seo_title, seo_description, + og_image, keywords, word_count, reading_time, slug)) + else: + db.execute(""" + INSERT INTO articles (title, slug, content_html, content_md, excerpt, + seo_title, seo_description, og_image, keywords, word_count, reading_time, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'published') + """, (title, slug, content_html, content_md, excerpt, seo_title, seo_description, + og_image, keywords, word_count, reading_time)) + + db.commit() + return jsonify({"status": "published", "slug": slug}), 201 + + +@app.route("/api/articles") +def api_articles(): + db = get_db() + limit = request.args.get("limit", 50) + rows = db.execute( + "SELECT * FROM articles WHERE status='published' ORDER BY published_at DESC LIMIT ?", + (limit,) + ).fetchall() + return jsonify([dict(r) for r in rows]) + + +@app.route("/api/stats") +def api_stats(): + db = get_db() + return jsonify({ + "total_articles": db.execute("SELECT COUNT(*) FROM articles WHERE status='published'").fetchone()[0], + "total_views": db.execute("SELECT COUNT(*) FROM pageviews").fetchone()[0], + "today_views": db.execute( + "SELECT COUNT(*) FROM pageviews WHERE created_at >= date('now')" + ).fetchone()[0], + "popular": get_popular(5), + }) + + +@app.route("/health") +def health(): + db = get_db() + count = db.execute("SELECT COUNT(*) FROM articles").fetchone()[0] + return jsonify({ + "status": "healthy", + "vertical": VERTICAL, + "articles": count, + "timestamp": datetime.now().isoformat(), + }) + + +# ─── Error Handlers ────────────────────────────────────────────── +@app.errorhandler(404) +def not_found(e): + return render_template_string(NOT_FOUND_TEMPLATE, **IDENTITY, domain=DOMAIN, vertical=VERTICAL), 404 + + +# ─── Templates ───────────────────────────────────────────────────── +HOME_TEMPLATE = """ + + + + + {{ name }} β€” {{ tagline }} + + + + + + + + + + + + + + +
    + +
    + +
    +

    {{ tagline }}

    +

    {{ description }}

    +
    + {{ total }} articles + {{ total_views }} readers + Updated daily +
    +
    + +
    + {% if popular %} +

    πŸ”₯ Trending Now

    +
    + {% for a in popular[:3] %} + +

    {{ a.title }}

    +

    {{ a.excerpt[:160] }}{% if a.excerpt|length > 160 %}...{% endif %}

    +
    + {{ a.views }} reads + {{ a.reading_time }} min + {{ a.published_at[:10] }} +
    +     + {% endfor %} +
    + {% endif %} + +

    πŸ“š Latest Articles

    +
    +
    + {% for a in articles %} + +

    {{ a.title }}

    +

    {{ a.excerpt[:200] }}{% if a.excerpt|length > 200 %}...{% endif %}

    +
    + {{ a.reading_time }} min read + {{ a.word_count }} words + {{ a.published_at[:10] }} +
    +     + {% endfor %} +
    + + {% if popular[3:] %} + + {% endif %} +
    +
    + + + + +""" + + +ARTICLE_TEMPLATE = """ + + + + + {{ article.seo_title or article.title }} + + + + + + + + + + + + +
    + +
    + +
    +
    +
    +

    {{ article.title }}

    +
    + {{ article.published_at[:10] }} + {{ article.reading_time }} min read + {{ article.word_count }} words +
    +
    + +
    + {{ article.content_html|safe }} +
    + +
    + {% if article.keywords %} +
    + {% for kw in article.keywords|from_json %} + {{ kw }} + {% endfor %} +
    + {% endif %} +
    +
    + + {% if related %} +
    +

    Continue Reading

    +
    + {% for r in related %} + +

    {{ r.title[:60] }}

    +
    {{ r.reading_time }} min Β· {{ r.published_at[:10] }}
    +     + {% endfor %} +
    +
    + {% endif %} +
    + + + + +""" + + +SEARCH_TEMPLATE = """ + + + + + Search β€” {{ name }} + + + + +
    + +
    +
    +

    Search {{ name }}

    +
    + + +
    + {% if results %} +

    {{ results|length }} results for "{{ query }}"

    + {% for r in results %} + +

    {{ r.title }}

    +

    {{ r.excerpt[:200] }}

    +     + {% endfor %} + {% elif query %} +

    No results for "{{ query }}"

    + {% endif %} +
    +
    Β© {{ domain }}
    + +""" + + +NOT_FOUND_TEMPLATE = """ + + + + 404 β€” {{ name }} + + + +
    +

    404

    +

    This page doesn't exist yet. Maybe it will tomorrow.

    + ← Back to {{ name }} +
    + +""" + + +# ─── Jinja filter ────────────────────────────────────────────────── +@app.template_filter("from_json") +def from_json_filter(s): + try: + return json.loads(s) + except (json.JSONDecodeError, TypeError): + return [] + + +# ─── Main ────────────────────────────────────────────────────────── +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=5000) + ap.add_argument("--host", type=str, default="0.0.0.0") + args = ap.parse_args() + + with app.app_context(): + init_db() + + print(f"πŸš€ {IDENTITY['name']} β†’ http://{args.host}:{args.port}") + app.run(host=args.host, port=args.port, debug=False) diff --git a/sites/_engine/start.sh b/sites/_engine/start.sh new file mode 100644 index 0000000..de46be2 --- /dev/null +++ b/sites/_engine/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Per-vertical site launcher β€” installed on each CT +# Usage: /opt/publisher/start.sh +# Expects PUBLISHER_VERTICAL env var to be set + +export PUBLISHER_VERTICAL="${PUBLISHER_VERTICAL:-guides}" +export PUBLISHER_SECRET="auto-publish-2026" + +cd /opt/publisher +exec python3 app.py --port 5000 --host 0.0.0.0