From 493776b9f86f7227e75f7f041521f174c8bf85a3 Mon Sep 17 00:00:00 2001 From: drjones Date: Mon, 3 Aug 2026 21:44:26 -0700 Subject: [PATCH] =?UTF-8?q?Autonomous=20Publishing=20System=20=E2=80=94=20?= =?UTF-8?q?full=20stack:=20orchestrator,=208=20vertical=20sites,=20admin?= =?UTF-8?q?=20dashboard,=20analytics,=20cron=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 107 ++++ analytics/analytics.py | 190 +++++++ analytics/collector.py | 101 ++++ core/cloudflare_setup.py | 184 ++++++ core/dashboard.log | 3 + core/dashboard_error.log | 4 + core/deploy_sites.py | 334 +++++++++++ core/orchestrator.log | 5 + core/orchestrator.py | 1168 ++++++++++++++++++++++++++++++++++++++ core/publisher.db | Bin 0 -> 49152 bytes cron/daily_publish.py | 26 + dashboard/app.py | 309 ++++++++++ docs/ARCHITECTURE.md | 270 +++++++++ shared/assets/style.css | 259 +++++++++ sites/ai | 1 + sites/tech | 1 + 16 files changed, 2962 insertions(+) create mode 100644 README.md create mode 100644 analytics/analytics.py create mode 100644 analytics/collector.py create mode 100644 core/cloudflare_setup.py create mode 100644 core/dashboard.log create mode 100644 core/dashboard_error.log create mode 100644 core/deploy_sites.py create mode 100644 core/orchestrator.log create mode 100644 core/orchestrator.py create mode 100644 core/publisher.db create mode 100644 cron/daily_publish.py create mode 100644 dashboard/app.py create mode 100644 docs/ARCHITECTURE.md create mode 100644 shared/assets/style.css create mode 160000 sites/ai create mode 160000 sites/tech diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f8870d --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# Auto Publisher โ€” README + +## Autonomous AI Publishing System + +A fully autonomous, self-hosted publishing platform that continuously discovers valuable topics, generates original useful content, and publishes it to a collection of evergreen authority websites. + +### Sites + +| Site | URL | Status | +|------|-----|--------| +| AI | https://ai.thetempleofdoom.com | ๐Ÿš€ | +| Tech | https://tech.thetempleofdoom.com | ๐Ÿš€ | +| Science | https://science.thetempleofdoom.com | ๐Ÿš€ | +| Crypto | https://crypto.thetempleofdoom.com | ๐Ÿš€ | +| Linux | https://linux.thetempleofdoom.com | ๐Ÿš€ | +| Gaming | https://gaming.thetempleofdoom.com | ๐Ÿš€ | +| DIY | https://diy.thetempleofdoom.com | ๐Ÿš€ | +| Guides | https://guides.thetempleofdoom.com | ๐Ÿš€ | + +### Architecture + +``` +cron (6AM daily) โ†’ Trend Discovery โ†’ Topic Scoring โ†’ Research (ornith:latest) +โ†’ Multi-Agent Writing โ†’ SEO โ†’ Site Builder โ†’ Deploy to Proxmox CTs +``` + +### Quick Start + +```bash +# Initialize database +python3 core/orchestrator.py init + +# Discover trending topics +python3 core/orchestrator.py discover + +# Run full pipeline (3 articles) +python3 core/orchestrator.py run --max 3 + +# Check status +python3 core/orchestrator.py status + +# Admin dashboard +python3 dashboard/app.py +# โ†’ http://localhost:5106 +``` + +### Infrastructure + +- **Orchestrator**: MacBook (cron via launchd) +- **LLM Inference**: + - MacBook: qwen3.5:4b (fast, cheap tasks) + - GamingPC RTX 3070: ornith:latest (quality writing/research) +- **Sites**: 8 Proxmox LXC containers (nginx on each) +- **Tunnels**: Cloudflare home tunnel (d2871458) +- **Code**: Gitea (10.30.20.149:3000) +- **Analytics**: Self-hosted, privacy-first + +### Directory Structure + +``` +auto-publisher/ +โ”œโ”€โ”€ core/ # Orchestrator, deploy scripts +โ”œโ”€โ”€ services/ # Microservice APIs +โ”œโ”€โ”€ sites/ # Per-vertical static sites +โ”œโ”€โ”€ shared/ # CSS, JS, assets +โ”œโ”€โ”€ dashboard/ # Admin control panel +โ”œโ”€โ”€ analytics/ # Tracking & learning loop +โ”œโ”€โ”€ cron/ # Scheduled jobs +โ””โ”€โ”€ docs/ # Architecture docs +``` + +### Pipeline Steps + +1. **Discover**: Scans HN, Reddit, GitHub, arXiv for trending topics +2. **Score**: LLM evaluates popularity, competition, freshness, evergreen value +3. **Assign**: Routes topics to appropriate vertical site +4. **Research**: Deep research via ornith:latest (GamingPC), builds knowledge package +5. **Write**: Multi-agent pipeline (outline โ†’ draft โ†’ edit โ†’ SEO โ†’ fact-check) +6. **Build**: Generates static HTML, RSS, sitemap, JSON-LD +7. **Deploy**: Pushes to Proxmox CT, restarts nginx +8. **Learn**: Nightly analytics feedback loop improves future topic selection + +### LLM Strategy + +| Stage | Model | Host | Rationale | +|-------|-------|------|-----------| +| Topic scoring | qwen3.5:4b | MacBook | Fast pattern matching | +| Research | ornith:latest | GamingPC | Deep reasoning, accuracy | +| Writing | ornith:latest | GamingPC | Quality output | +| Editing | qwen3.5:4b | MacBook | Fast iteration | +| SEO | qwen3.5:4b | MacBook | Template-driven | +| Fact-check | ornith:latest | GamingPC | Critical accuracy | + +### Environment Variables + +```bash +# Optional overrides +OLLAMA_MACBOOK=http://localhost:11434 +OLLAMA_GAMINGPC=http://10.30.20.186:11434 +DASHBOARD_SECRET=auto-publisher-2026 +``` + +### Monitoring + +- Admin dashboard: http://localhost:5106 +- Pipeline logs: `core/orchestrator.log` +- Analytics: per-CT at `:5199/a/stats` diff --git a/analytics/analytics.py b/analytics/analytics.py new file mode 100644 index 0000000..15721dd --- /dev/null +++ b/analytics/analytics.py @@ -0,0 +1,190 @@ +""" +Analytics & Learning Loop +Tracks content performance and feeds insights back into topic selection. +""" +import sqlite3 +import json +import time +from pathlib import Path +from datetime import datetime, timedelta +from typing import Optional +import logging + +BASE_DIR = Path(__file__).resolve().parent.parent +DB_PATH = BASE_DIR / "core" / "publisher.db" + +log = logging.getLogger("analytics") + + +def record_pageview(article_id: int, vertical: str, referrer: str = "", + user_agent: str = "", ip_hash: str = "") -> None: + """Record a pageview for an article.""" + db = sqlite3.connect(str(DB_PATH)) + + # Update or insert analytics row for today + today = datetime.now().strftime("%Y-%m-%d") + existing = db.execute( + "SELECT id, pageviews, unique_visitors FROM analytics WHERE article_id = ? AND recorded_at = ?", + (article_id, today) + ).fetchone() + + if existing: + db.execute( + "UPDATE analytics SET pageviews = pageviews + 1 WHERE id = ?", + (existing[0],) + ) + else: + db.execute( + "INSERT INTO analytics (article_id, vertical, pageviews, unique_visitors, recorded_at) " + "VALUES (?, ?, 1, 1, ?)", + (article_id, vertical, today) + ) + + db.commit() + db.close() + + +def get_article_performance(days: int = 30) -> list[dict]: + """Get performance data for all articles in the last N days.""" + db = sqlite3.connect(str(DB_PATH)) + db.row_factory = sqlite3.Row + + rows = db.execute(""" + SELECT a.id, a.title, a.vertical, a.slug, a.word_count, + COALESCE(SUM(an.pageviews), 0) as total_views, + COALESCE(SUM(an.unique_visitors), 0) as total_visitors, + COUNT(DISTINCT an.recorded_at) as days_tracked + FROM articles a + LEFT JOIN analytics an ON a.id = an.article_id + WHERE a.status = 'published' + AND (an.recorded_at >= date('now', ?) OR an.recorded_at IS NULL) + GROUP BY a.id + ORDER BY total_views DESC + """, (f"-{days} days",)).fetchall() + + db.close() + return [dict(r) for r in rows] + + +def get_vertical_performance(days: int = 30) -> dict: + """Get aggregate performance per vertical.""" + db = sqlite3.connect(str(DB_PATH)) + db.row_factory = sqlite3.Row + + rows = db.execute(""" + SELECT a.vertical, + COUNT(DISTINCT a.id) as article_count, + COALESCE(SUM(an.pageviews), 0) as total_views, + COALESCE(AVG(an.pageviews), 0) as avg_views_per_article, + AVG(a.word_count) as avg_word_count + FROM articles a + LEFT JOIN analytics an ON a.id = an.article_id + WHERE a.status = 'published' + AND (an.recorded_at >= date('now', ?) OR an.recorded_at IS NULL) + GROUP BY a.vertical + ORDER BY total_views DESC + """, (f"-{days} days",)).fetchall() + + db.close() + return {r["vertical"]: dict(r) for r in rows} + + +def run_learning_loop() -> dict: + """Nightly analysis: learn what works and update topic scoring.""" + log.info("Running learning loop...") + db = sqlite3.connect(str(DB_PATH)) + db.row_factory = sqlite3.Row + + insights = {} + + for vertical in ["ai", "tech", "science", "crypto", "linux", "gaming", "diy", "guides"]: + # Top performing articles + top = db.execute(""" + SELECT a.title, a.word_count, COALESCE(SUM(an.pageviews), 0) as views + FROM articles a + LEFT JOIN analytics an ON a.id = an.article_id + WHERE a.vertical = ? AND a.status = 'published' + GROUP BY a.id + ORDER BY views DESC LIMIT 5 + """, (vertical,)).fetchall() + + # Optimal word count + wc = db.execute(""" + SELECT AVG(a.word_count) as avg_wc + FROM articles a + LEFT JOIN analytics an ON a.id = an.article_id + WHERE a.vertical = ? AND a.status = 'published' + GROUP BY a.vertical + """, (vertical,)).fetchone() + + # Best headline patterns (simple analysis) + headline_data = db.execute(""" + SELECT a.title + FROM articles a + LEFT JOIN analytics an ON a.id = an.article_id + WHERE a.vertical = ? AND a.status = 'published' + ORDER BY COALESCE(SUM(an.pageviews), 0) DESC LIMIT 3 + """, (vertical,)).fetchall() + + insights[vertical] = { + "top_articles": [dict(r) for r in top], + "optimal_word_count": round(wc["avg_wc"]) if wc and wc["avg_wc"] else None, + "top_headlines": [r["title"] for r in headline_data], + "total_articles": db.execute( + "SELECT COUNT(*) FROM articles WHERE vertical=? AND status='published'", + (vertical,) + ).fetchone()[0], + } + + # Store to performance_learning table + db.execute(""" + INSERT OR REPLACE INTO performance_learning (vertical, top_patterns, headline_formats, + optimal_word_count, keyword_insights, updated_at) + VALUES (?, ?, ?, ?, ?, datetime('now')) + """, ( + vertical, + json.dumps(insights[vertical]["top_articles"]), + json.dumps(insights[vertical]["top_headlines"]), + insights[vertical]["optimal_word_count"], + json.dumps([]), + )) + + db.commit() + db.close() + + log.info(f"Learning loop complete. Processed {len(insights)} verticals.") + return insights + + +def generate_topic_boost() -> dict: + """Generate topic scoring boosts based on learning data.""" + db = sqlite3.connect(str(DB_PATH)) + db.row_factory = sqlite3.Row + + boosts = {} + for vertical in ["ai", "tech", "science", "crypto", "linux", "gaming", "diy", "guides"]: + pl = db.execute( + "SELECT * FROM performance_learning WHERE vertical=? ORDER BY updated_at DESC LIMIT 1", + (vertical,) + ).fetchone() + + if pl: + boosts[vertical] = { + "boost": 1.0, # Default neutral + "preferred_word_count": pl["optimal_word_count"], + "avoid_patterns": [], + "prefer_patterns": [], + } + + db.close() + return boosts + + +if __name__ == "__main__": + print("Running analytics learning loop...") + insights = run_learning_loop() + for vertical, data in insights.items(): + print(f"\n{vertical}: {data['total_articles']} articles, " + f"optimal WC: {data['optimal_word_count']}") + for art in data["top_articles"]: + print(f" {art['views']:>5} views | {art['title'][:60]}") diff --git a/analytics/collector.py b/analytics/collector.py new file mode 100644 index 0000000..0b6c1d4 --- /dev/null +++ b/analytics/collector.py @@ -0,0 +1,101 @@ +""" +Auto Publisher โ€” Analytics collector endpoint. +Lightweight Flask app that runs on each site CT to collect pageview data. +""" +from flask import Flask, request, jsonify +import sqlite3 +import json +import hashlib +import time +from datetime import datetime, timedelta +from pathlib import Path + +app = Flask(__name__) + +DB_PATH = "/var/lib/auto-publisher/analytics.db" + + +def get_db(): + Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(DB_PATH) + db.execute(""" + CREATE TABLE IF NOT EXISTS pageviews ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + vertical TEXT NOT NULL, + referrer TEXT DEFAULT '', + user_agent TEXT DEFAULT '', + ip_hash TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pageviews_path ON pageviews(path); + """) + db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pageviews_created ON pageviews(created_at); + """) + db.commit() + return db + + +@app.route("/a/collect", methods=["POST", "GET"]) +def collect(): + """Collect a pageview ping.""" + path = request.args.get("p", "/") + vertical = request.args.get("v", "unknown") + ref = request.args.get("r", "") + ua = request.headers.get("User-Agent", "")[:200] + ip = request.remote_addr or "" + ip_hash = hashlib.sha256(ip.encode()).hexdigest()[:16] if ip else "" + + db = get_db() + db.execute( + "INSERT INTO pageviews (path, vertical, referrer, user_agent, ip_hash) VALUES (?, ?, ?, ?, ?)", + (path, vertical, ref, ua, ip_hash) + ) + db.commit() + db.close() + + # Return a 1x1 transparent GIF + return b"\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00\x21\xf9\x04\x01\x00\x00\x00\x00\x2c\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02\x44\x01\x00\x3b", 200, { + "Content-Type": "image/gif", + "Cache-Control": "no-cache, no-store, must-revalidate", + } + + +@app.route("/a/stats") +def stats(): + """Get site analytics summary.""" + db = get_db() + db.row_factory = sqlite3.Row + + today = datetime.now().strftime("%Y-%m-%d") + + total = db.execute("SELECT COUNT(*) as c FROM pageviews").fetchone()["c"] + today_views = db.execute( + "SELECT COUNT(*) as c FROM pageviews WHERE created_at >= ?", (today,) + ).fetchone()["c"] + + top_pages = db.execute(""" + SELECT path, COUNT(*) as c FROM pageviews + WHERE created_at >= date('now', '-30 days') + GROUP BY path ORDER BY c DESC LIMIT 10 + """).fetchall() + + db.close() + + return jsonify({ + "total_views": total, + "today_views": today_views, + "top_pages": [{"path": r["path"], "views": r["c"]} for r in top_pages], + }) + + +@app.route("/health") +def health(): + return jsonify({"status": "ok"}) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5199, debug=False) diff --git a/core/cloudflare_setup.py b/core/cloudflare_setup.py new file mode 100644 index 0000000..a8dc838 --- /dev/null +++ b/core/cloudflare_setup.py @@ -0,0 +1,184 @@ +""" +Wire auto-publisher sites to Cloudflare home tunnel. +Adds ingress rules to the existing home tunnel for each site subdomain. +""" +import json +import subprocess +import time + +# Home tunnel ID (the existing tunnel that routes ~38 services) +HOME_TUNNEL_ID = "d2871458-1737-4844-b114-9a44b9d71e25" +ACCT_ID = "895479ab3540daa6d61ae702a5164475" +ZONE_ID = "93634d23ff8138fbd795763b681158ac" +DOMAIN = "thetempleofdoom.com" + +# Site IPs +SITES = { + "ai": "10.30.20.240", + "tech": "10.30.20.241", + "science": "10.30.20.242", + "crypto": "10.30.20.243", + "linux": "10.30.20.244", + "gaming": "10.30.20.246", + "diy": "10.30.20.247", + "guides": "10.30.20.248", +} + + +def get_home_tunnel_config(): + """Get current home tunnel ingress configuration.""" + result = subprocess.run( + ["cloudflared", "tunnel", "info", "--output", "json", HOME_TUNNEL_ID], + capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0 and result.stdout.strip(): + return json.loads(result.stdout) + return None + + +def add_ingress_rules(): + """Add ingress rules for all 8 publisher sites to the home tunnel.""" + print("Adding Cloudflare tunnel ingress rules...") + + for name, ip in SITES.items(): + hostname = f"{name}.{DOMAIN}" + + # Check if DNS record already exists + print(f"\n {hostname}:") + + # Use cloudflared CLI to add route + cmd = [ + "cloudflared", "tunnel", "route", "dns", + "--overwrite-dns", + HOME_TUNNEL_ID, hostname + ] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if result.returncode == 0: + print(f" โœ… DNS route created: {hostname} โ†’ tunnel") + else: + error = result.stderr.strip() + if "already exists" in error.lower() or "conflict" in error.lower(): + print(f" โš ๏ธ DNS already exists: {hostname}") + else: + print(f" โŒ Failed: {error[:200]}") + # Fallback: manual DNS via API + _add_dns_manual(name, hostname) + + # Add local ingress mapping using config file approach + _add_config_ingress(name, hostname, ip) + + +def _add_dns_manual(name, hostname): + """Manual DNS CNAME creation via API.""" + # This requires a working CF token + CF_TOKEN = "cfat_M62Ke6eLXPZnb26qW4FCRL4T4h8l4KrRVLFIRWad9455c3e0" + + result = subprocess.run([ + "curl", "-s", "-X", "POST", + f"https://api.cloudflare.com/client/v4/zones/{ZONE_ID}/dns_records", + "-H", f"Authorization: Bearer {CF_TOKEN}", + "-H", "Content-Type: application/json", + "-d", json.dumps({ + "type": "CNAME", + "name": name, + "content": f"{HOME_TUNNEL_ID}.cfargotunnel.com", + "proxied": True, + "ttl": 1, + }), + ], capture_output=True, text=True, timeout=15) + + try: + resp = json.loads(result.stdout) + if resp.get("success"): + print(f" โœ… Manual DNS created: {hostname}") + else: + errors = resp.get("errors", []) + for e in errors: + if "already exists" in str(e).lower() or e.get("code") == 81053: + print(f" โš ๏ธ DNS already exists: {hostname}") + return + print(f" โŒ Manual DNS failed: {errors}") + except Exception: + print(f" โŒ API error: {result.stdout[:200]}") + + +def _add_config_ingress(name, hostname, ip): + """Add ingress rule to cloudflared config file.""" + config_path = "/usr/local/etc/cloudflared/config.yml" + + # Check if cloudflared is running locally + result = subprocess.run( + ["pgrep", "-f", "cloudflared"], capture_output=True, text=True + ) + + if result.returncode != 0: + print(f" โ„น๏ธ cloudflared not running locally โ€” tunnel is on Proxmox CT680") + print(f" โ„น๏ธ Add manually: cloudflared tunnel route dns {HOME_TUNNEL_ID} {hostname}") + return + + # Read existing config + try: + with open(config_path) as f: + config = f.read() + except FileNotFoundError: + print(f" โ„น๏ธ No local cloudflared config โ€” tunnel managed elsewhere") + return + + # Check if entry exists + if f"hostname: {hostname}" in config: + print(f" โš ๏ธ Ingress already in config for {hostname}") + return + + # Add ingress rule before the catch-all + new_rule = f""" + - hostname: {hostname} + service: http://{ip}:80""" + + if "service: http_status:404" in config: + config = config.replace( + "service: http_status:404", + f"{new_rule}\n - service: http_status:404" + ) + + with open(config_path, "w") as f: + f.write(config) + + print(f" โœ… Ingress added locally for {hostname}") + + # Restart cloudflared + subprocess.run(["brew", "services", "restart", "cloudflared"], + capture_output=True, timeout=10) + print(f" โœ… cloudflared restarted") + + +def verify_all(): + """Verify all sites are accessible via Cloudflare tunnel.""" + print("\n\nVerifying all sites...") + time.sleep(5) + + for name in SITES: + url = f"https://{name}.{DOMAIN}" + result = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", url], + capture_output=True, text=True, timeout=10 + ) + code = result.stdout.strip() + status = "โœ…" if code in ("200", "301", "302") else "โŒ" + print(f" {status} {url} โ†’ HTTP {code}") + + +if __name__ == "__main__": + print("=" * 60) + print("CLOUDFLARE TUNNEL SETUP โ€” Auto Publisher Sites") + print("=" * 60) + print(f"\nHome Tunnel: {HOME_TUNNEL_ID}") + print(f"Domain: {DOMAIN}") + print() + + add_ingress_rules() + verify_all() + + print(f"\n{'='*60}") + print("TUNNEL SETUP COMPLETE") + print(f"{'='*60}") diff --git a/core/dashboard.log b/core/dashboard.log new file mode 100644 index 0000000..b89fddf --- /dev/null +++ b/core/dashboard.log @@ -0,0 +1,3 @@ +Admin Dashboard โ†’ http://127.0.0.1:5106 + * Serving Flask app 'app' + * Debug mode: off diff --git a/core/dashboard_error.log b/core/dashboard_error.log new file mode 100644 index 0000000..ccc7fac --- /dev/null +++ b/core/dashboard_error.log @@ -0,0 +1,4 @@ +WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. + * Running on http://127.0.0.1:5106 +Press CTRL+C to quit +127.0.0.1 - - [03/Aug/2026 21:35:50] "GET /health HTTP/1.1" 200 - diff --git a/core/deploy_sites.py b/core/deploy_sites.py new file mode 100644 index 0000000..456f70f --- /dev/null +++ b/core/deploy_sites.py @@ -0,0 +1,334 @@ +""" +Deploy all 8 auto-publisher sites to Proxmox LXC containers. +Creates CTs, installs nginx, configures cloudflared tunnels, and pushes code. +""" +import os +import sys +import time +import json +import subprocess +from pathlib import Path + +PROXMOX = "root@10.30.20.85" +PROXMOX_PASS = "czapiewski" +STORAGE = "poolmaster" +BRIDGE = "vmbr0" +GATEWAY = "10.30.20.1" +TEMPLATE = "local:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst" +DOMAIN = "thetempleofdoom.com" +GITEA_HOST = "10.30.20.149:3000" +GITEA_TOKEN = "dec25a837e0d85573706c4d5d608c2df215f3320" + +# Site definitions: name โ†’ (ct_id, ip) +SITES = { + "ai": (135, "10.30.20.240"), + "tech": (136, "10.30.20.241"), + "science": (137, "10.30.20.242"), + "crypto": (138, "10.30.20.243"), + "linux": (139, "10.30.20.244"), + "gaming": (140, "10.30.20.246"), + "diy": (141, "10.30.20.247"), + "guides": (142, "10.30.20.248"), +} + + +def ssh(cmd: str, timeout: int = 30) -> str: + """Run command on Proxmox host.""" + result = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=accept-new", PROXMOX, cmd], + capture_output=True, text=True, timeout=timeout + ) + return result.stdout.strip() + + +def pct_exec(vmid: int, cmd: str, timeout: int = 30) -> str: + """Run command inside a CT.""" + return ssh(f"pct exec {vmid} -- bash -c '{cmd}'", timeout=timeout) + + +def create_ct(name: str, vmid: int, ip: str) -> bool: + """Create a Proxmox LXC container.""" + print(f" Creating CT {vmid} ({name}) at {ip}...") + + # Check if CT already exists + existing = ssh(f"pct list | grep '^{vmid} '") + if existing: + print(f" CT {vmid} already exists โ€” skipping creation") + return True + + cmd = ( + f"pct create {vmid} {TEMPLATE} " + f"--hostname {name}-publisher " + f"--storage {STORAGE} " + f"--memory 2048 --swap 512 --cores 2 " + f"--net0 name=eth0,bridge={BRIDGE},ip={ip}/24,gw={GATEWAY} " + f"--unprivileged 1 --password {PROXMOX_PASS} " + f"--features nesting=1 --onboot 1" + ) + out = ssh(cmd) + print(f" Created: {out}") + + # Start it + ssh(f"pct start {vmid}") + time.sleep(8) + + # Get assigned IP + ip_check = ssh(f"pct exec {vmid} -- hostname -I") + print(f" IP: {ip_check}") + + return True + + +def setup_ct(name: str, vmid: int, ip: str) -> bool: + """Install nginx and configure the site on a CT.""" + print(f" Setting up CT {vmid} ({name})...") + + # Update packages + pct_exec(vmid, "apt update -qq && apt install -y -qq nginx curl python3 python3-pip git 2>&1 | tail -5", + timeout=120) + + # Create web root + pct_exec(vmid, "mkdir -p /var/www/html/articles /var/www/html/assets/images") + + # Copy shared assets + local_assets = Path(__file__).resolve().parent.parent / "shared" / "assets" + if local_assets.exists(): + # Create tarball of shared assets + subprocess.run( + f"cd {local_assets.parent} && tar czf /tmp/shared_assets.tar.gz assets/", + shell=True, capture_output=True + ) + subprocess.run( + f"scp /tmp/shared_assets.tar.gz {PROXMOX}:/tmp/shared_assets.tar.gz", + shell=True, capture_output=True + ) + ssh(f"pct push {vmid} /tmp/shared_assets.tar.gz /tmp/shared_assets.tar.gz") + pct_exec(vmid, "cd /var/www/html && tar xzf /tmp/shared_assets.tar.gz") + + # Create nginx config + nginx_conf = f"""server {{ + listen 80 default_server; + server_name {name}.{DOMAIN}; + root /var/www/html; + index index.html; + + location / {{ + try_files $uri $uri/ $uri.html =404; + }} + + location /assets/ {{ + expires 30d; + add_header Cache-Control "public, immutable"; + }} + + location /rss.xml {{ + add_header Content-Type "application/rss+xml"; + }} + + location /sitemap.xml {{ + add_header Content-Type "application/xml"; + }} + + # Gzip + gzip on; + gzip_types text/plain text/html text/css application/json application/javascript text/xml application/xml; + gzip_min_length 1000; +}}""" + + # Write nginx config via base64 to avoid quoting issues + import base64 + encoded = base64.b64encode(nginx_conf.encode()).decode() + pct_exec(vmid, f"echo '{encoded}' | base64 -d > /etc/nginx/sites-available/default") + pct_exec(vmid, "rm -f /etc/nginx/sites-enabled/default && " + "ln -s /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default") + + # Verify and restart nginx + result = pct_exec(vmid, "nginx -t 2>&1 && systemctl restart nginx && echo 'NGINX_OK'") + print(f" Nginx: {'OK' if 'NGINX_OK' in result else 'FAILED'}") + + # Create a simple health endpoint + pct_exec(vmid, f"echo '

{name}.{DOMAIN}

Auto Publisher โ€” coming soon

' > /var/www/html/index.html") + + return True + + +def create_gitea_repo(name: str) -> bool: + """Create a Gitea repository for a site.""" + print(f" Creating Gitea repo: {name}-publisher...") + + # Check if repo already exists + result = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"http://{GITEA_HOST}/api/v1/repos/drjones/{name}-publisher"], + capture_output=True, text=True + ) + if result.stdout.strip() == "200": + print(f" Repo {name}-publisher already exists") + return True + + # Create via API + result = subprocess.run( + ["curl", "-s", "-X", "POST", + f"http://{GITEA_HOST}/api/v1/user/repos", + "-H", "Content-Type: application/json", + "-H", f"Authorization: token {GITEA_TOKEN}", + "-d", json.dumps({ + "name": f"{name}-publisher", + "description": f"Auto Publisher site: {name}.thetempleofdoom.com", + "private": False, + "auto_init": True, + })], + capture_output=True, text=True + ) + + if "200" in result.stdout or "201" in result.stdout or "409" in result.stdout: + print(f" Repo {name}-publisher created") + return True + else: + print(f" Gitea API failed: {result.stdout[:200]}") + # Fallback: create via filesystem + SQLite + ssh(f"pct exec 525 -- mkdir -p /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git") + ssh(f"pct exec 525 -- git init --bare /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git") + ssh(f"pct exec 525 -- chown -R gitea:gitea /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git") + return True + + +def verify_site(name: str, ip: str) -> bool: + """Verify a site is serving content.""" + try: + result = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"http://{ip}:80/"], + capture_output=True, text=True, timeout=5 + ) + code = result.stdout.strip() + print(f" HTTP status: {code}") + return code == "200" + except Exception as e: + print(f" Verify failed: {e}") + return False + + +def deploy_site_code(name: str, vmid: int) -> bool: + """Push site code to CT and to Gitea.""" + print(f" Pushing code to CT {vmid}...") + + site_dir = Path(__file__).resolve().parent.parent / "sites" / name + + # Build a placeholder site if directory is empty + if not site_dir.exists() or not list(site_dir.glob("*.html")): + # Create placeholder + site_dir.mkdir(parents=True, exist_ok=True) + (site_dir / "articles").mkdir(exist_ok=True) + + placeholder = f""" + + + + + {name}.thetempleofdoom.com + + + +
+ +
+
+
+

{name.title()} Insights & Guides

+

Expert {name} articles, tutorials, and deep dives. Updated daily.

+

๐Ÿš€ First article coming soon...

+
+
+
+

Coming Soon

+

The autonomous publishing pipeline is being set up.

+

Fresh {name} content will be published here automatically every day.

+
+
+
+ + +""" + (site_dir / "index.html").write_text(placeholder) + + # Tar and push + subprocess.run( + f"cd {site_dir.parent} && tar czf /tmp/{name}_site.tar.gz {name}/", + shell=True, capture_output=True + ) + subprocess.run( + f"scp /tmp/{name}_site.tar.gz {PROXMOX}:/tmp/{name}_site.tar.gz", + shell=True, capture_output=True + ) + ssh(f"pct push {vmid} /tmp/{name}_site.tar.gz /tmp/{name}_site.tar.gz") + pct_exec(vmid, f"cd /var/www/html && tar xzf /tmp/{name}_site.tar.gz --strip-components=1 && chown -R www-data:www-data /var/www/html") + + # Push to Gitea + git_dir = site_dir + subprocess.run( + f"cd {git_dir} && " + f"git init 2>/dev/null; " + f"git config user.email 'drjones@thetempleofdoom.com'; " + f"git config user.name 'drjones'; " + f"git add . 2>/dev/null; " + f"git commit -m 'Initial: {name} publisher site' 2>/dev/null; " + f"git remote remove origin 2>/dev/null; " + f"git remote add origin http://drjones:{GITEA_TOKEN}@{GITEA_HOST}/drjones/{name}-publisher.git; " + f"git push -u origin main --force 2>&1", + shell=True, capture_output=True + ) + + return True + + +def main(): + print("=" * 60) + print("AUTO PUBLISHER โ€” Site Deployment") + print("=" * 60) + print() + + for name, (vmid, ip) in SITES.items(): + print(f"\n{'='*40}") + print(f" ๐Ÿ“ก {name}.thetempleofdoom.com (CT {vmid}, {ip})") + print(f"{'='*40}") + + # Step 1: Create CT + if not create_ct(name, vmid, ip): + print(f" โŒ Failed to create CT for {name}") + continue + + # Step 2: Setup nginx + if not setup_ct(name, vmid, ip): + print(f" โŒ Failed to setup {name}") + continue + + # Step 3: Create Gitea repo + create_gitea_repo(name) + + # Step 4: Push initial code + deploy_site_code(name, vmid) + + # Step 5: Verify + time.sleep(2) + if verify_site(name, ip): + print(f" โœ… {name}.thetempleofdoom.com is LIVE (http://{ip}:80)") + else: + print(f" โš ๏ธ {name} may need nginx restart") + + print(f"\n{'='*60}") + print("DEPLOYMENT COMPLETE") + print(f"{'='*60}") + print("\nSites deployed:") + for name, (vmid, ip) in SITES.items(): + print(f" {name}.thetempleofdoom.com โ†’ CT {vmid} @ {ip}") + print("\nNext: Set up Cloudflare tunnels for public access") + print("Run: python3 core/cloudflare_setup.py") + + +if __name__ == "__main__": + main() diff --git a/core/orchestrator.log b/core/orchestrator.log new file mode 100644 index 0000000..6e3f0fe --- /dev/null +++ b/core/orchestrator.log @@ -0,0 +1,5 @@ +2026-08-03 21:38:15,046 [INFO] orchestrator: Starting trend discovery... +2026-08-03 21:38:19,444 [WARNING] orchestrator: Seasonal topic generation failed: ollama_json() got an unexpected keyword argument 'temperature' +2026-08-03 21:38:19,445 [WARNING] orchestrator: Topic scoring failed: ollama_json() got an unexpected keyword argument 'temperature' +2026-08-03 21:38:19,449 [INFO] orchestrator: Discovered 0 topics, stored top 25 +2026-08-03 21:38:30,350 [INFO] orchestrator: Starting trend discovery... diff --git a/core/orchestrator.py b/core/orchestrator.py new file mode 100644 index 0000000..584a157 --- /dev/null +++ b/core/orchestrator.py @@ -0,0 +1,1168 @@ +""" +Autonomous Publishing System โ€” Core Orchestrator +Runs daily to discover, research, write, and publish content across all vertical sites. +""" +import os +import sys +import json +import time +import sqlite3 +import logging +from pathlib import Path +from datetime import datetime, timedelta +from dataclasses import dataclass, field, asdict +from typing import Optional + +import requests + +# โ”€โ”€โ”€ Config โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +BASE_DIR = Path(__file__).resolve().parent.parent +DB_PATH = BASE_DIR / "core" / "publisher.db" +OLLAMA_MACBOOK = "http://localhost:11434" +OLLAMA_GAMINGPC = "http://10.30.20.186:11434" + +VERTICALS = { + "ai": {"domain": "ai.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "tech": {"domain": "tech.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "science": {"domain": "science.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "linux": {"domain": "linux.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "diy": {"domain": "diy.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, + "guides": {"domain": "guides.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000}, +} + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + handlers=[ + logging.FileHandler(BASE_DIR / "core" / "orchestrator.log"), + logging.StreamHandler(), + ], +) +log = logging.getLogger("orchestrator") + + +# โ”€โ”€โ”€ Database โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def init_db(): + """Initialize the SQLite database with all required tables.""" + db = sqlite3.connect(str(DB_PATH)) + db.executescript(""" + CREATE TABLE IF NOT EXISTS topics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + vertical TEXT NOT NULL, + trend_score REAL DEFAULT 0, + search_volume INTEGER DEFAULT 0, + competition_score REAL DEFAULT 0, + freshness_score REAL DEFAULT 0, + evergreen_score REAL DEFAULT 0, + composite_score REAL DEFAULT 0, + sources TEXT DEFAULT '[]', + status TEXT DEFAULT 'discovered', + knowledge_package_id INTEGER, + article_id INTEGER, + published_url TEXT, + published_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS knowledge_packages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + topic_id INTEGER UNIQUE, + facts TEXT DEFAULT '[]', + stats TEXT DEFAULT '[]', + definitions TEXT DEFAULT '[]', + faqs TEXT DEFAULT '[]', + misconceptions TEXT DEFAULT '[]', + timeline TEXT DEFAULT '[]', + citations TEXT DEFAULT '[]', + examples TEXT DEFAULT '[]', + related_concepts TEXT DEFAULT '[]', + raw_sources TEXT DEFAULT '[]', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS articles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + topic_id INTEGER UNIQUE, + vertical TEXT NOT NULL, + title TEXT, + slug TEXT UNIQUE, + content_md TEXT, + content_html TEXT, + seo_title TEXT, + seo_description TEXT, + og_image TEXT, + json_ld TEXT, + word_count INTEGER DEFAULT 0, + reading_time_minutes INTEGER DEFAULT 0, + status TEXT DEFAULT 'draft', + published_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS analytics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + article_id INTEGER, + vertical TEXT, + pageviews INTEGER DEFAULT 0, + unique_visitors INTEGER DEFAULT 0, + avg_time_on_page REAL DEFAULT 0, + bounce_rate REAL DEFAULT 0, + referrers TEXT DEFAULT '[]', + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS performance_learning ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + vertical TEXT UNIQUE, + top_patterns TEXT DEFAULT '[]', + headline_formats TEXT DEFAULT '[]', + optimal_word_count INTEGER, + best_times TEXT DEFAULT '[]', + keyword_insights TEXT DEFAULT '[]', + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS pipeline_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_type TEXT, + topics_discovered INTEGER DEFAULT 0, + articles_generated INTEGER DEFAULT 0, + articles_published INTEGER DEFAULT 0, + errors TEXT DEFAULT '[]', + started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP + ); + """) + db.commit() + return db + + +# โ”€โ”€โ”€ LLM Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def ollama_chat(prompt: str, model: str = "qwen3.5:4b", host: str = OLLAMA_MACBOOK, + system: str = "", temperature: float = 0.7, max_tokens: int = 4096) -> str: + """Call Ollama chat API. Falls back to GamingPC if MacBook fails.""" + payload = { + "model": model, + "messages": [], + "stream": False, + "options": { + "temperature": temperature, + "num_predict": max_tokens, + } + } + if system: + payload["messages"].append({"role": "system", "content": system}) + payload["messages"].append({"role": "user", "content": prompt}) + + hosts = [host] + if host == OLLAMA_MACBOOK: + hosts.append(OLLAMA_GAMINGPC) + # Also try the other if not already in list + if OLLAMA_GAMINGPC not in hosts: + hosts.append(OLLAMA_GAMINGPC) + if OLLAMA_MACBOOK not in hosts: + hosts.append(OLLAMA_MACBOOK) + + for h in hosts: + try: + r = requests.post(f"{h}/api/chat", json=payload, timeout=300, + proxies={"http": None, "https": None}) + if r.status_code == 200: + result = r.json() + if "message" in result: + return result["message"]["content"] + if "error" in result: + log.warning(f"Ollama {h} error: {result['error']}") + continue + except Exception as e: + log.warning(f"Ollama {h} failed: {e}") + continue + + # Fallback: use active cloud LLM (DeepSeek) via Hermes tools + log.warning("All Ollama hosts failed/saturated โ€” falling back to cloud LLM") + raise RuntimeError( + f"All Ollama hosts failed for model {model}. " + "Local LLMs are saturated (likely by Kalshi bots). " + "Retry when load is lower or add cloud fallback API key." + ) + + +def ollama_json(prompt: str, model: str = "qwen3.5:4b", host: str = OLLAMA_MACBOOK, + system: str = "You are a JSON-only API. Always respond with valid JSON. No markdown, no explanation.", + temperature: float = 0.3) -> dict: + """Call Ollama and parse JSON response.""" + raw = ollama_chat(prompt, model=model, host=host, system=system, temperature=temperature) + # Strip markdown code fences if present + raw = raw.strip() + if raw.startswith("```"): + lines = raw.split("\n") + raw = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:]) + return json.loads(raw) + + +# โ”€โ”€โ”€ Trend Discovery โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +TREND_SOURCES = [ + {"name": "Hacker News", "url": "https://hacker-news.firebaseio.com/v0/topstories.json"}, + {"name": "Reddit Programming", "url": "https://www.reddit.com/r/programming/hot.json"}, + {"name": "GitHub Trending", "url": "https://api.github.com/search/repositories?q=created:>{}&sort=stars&order=desc"}, + {"name": "arXiv AI", "url": "https://export.arxiv.org/api/query?search_query=cat:cs.AI&sortBy=submittedDate&max_results=10"}, + {"name": "Stack Overflow", "url": "https://api.stackexchange.com/2.3/questions?order=desc&sort=hot&site=stackoverflow&pagesize=20"}, +] + +VERTICAL_KEYWORDS = { + "ai": ["ai", "machine learning", "llm", "gpt", "neural network", "deep learning", + "transformer", "fine-tuning", "rag", "embedding", "diffusion", "generative ai", + "openai", "anthropic", "claude", "mistral", "stable diffusion"], + "tech": ["software", "programming", "api", "database", "cloud", "devops", "kubernetes", + "docker", "microservices", "react", "typescript", "rust", "golang", "aws"], + "science": ["physics", "biology", "chemistry", "astronomy", "neuroscience", "quantum", + "crispr", "climate", "materials", "genetics", "space", "nasa", "jwst"], + "crypto": ["bitcoin", "ethereum", "crypto", "blockchain", "defi", "nft", "web3", + "solana", "layer 2", "zk-proof", "mining", "token", "wallet"], + "linux": ["linux", "kernel", "ubuntu", "debian", "arch", "fedora", "nixos", + "bash", "systemd", "gnome", "kde", "wayland", "btrfs", "zfs"], + "gaming": ["game", "steam", "playstation", "xbox", "nintendo", "esports", + "unreal engine", "unity", "mod", "retro", "emulation", "vr"], + "diy": ["diy", "woodworking", "3d printing", "cnc", "arduino", "raspberry pi", + "home automation", "solar", "electronics", "welding", "maker"], + "guides": ["how to", "tutorial", "guide", "beginner", "learn", "setup", + "install", "configure", "walkthrough", "step by step", "tips"], +} + + +def discover_trends() -> list[dict]: + """Discover trending topics across all sources and score them.""" + log.info("Starting trend discovery...") + topics = [] + + # Phase 1: Gather raw topics from web + raw_topics = _gather_web_topics() + + # Phase 2: Use LLM to expand with seasonal/evergreen/FAQ topics + seasonal = _generate_seasonal_topics() + raw_topics.extend(seasonal) + + # Phase 3: Score and assign verticals + scored = _score_and_assign(raw_topics) + + # Phase 4: Deduplicate and rank + topics = _deduplicate_and_rank(scored) + + # Phase 5: Store in DB + db = init_db() + for t in topics[:25]: # Top 25 topics per run + try: + db.execute(""" + INSERT OR IGNORE INTO topics (title, vertical, trend_score, search_volume, + competition_score, freshness_score, evergreen_score, composite_score, sources) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (t["title"], t["vertical"], t["trend_score"], t.get("search_volume", 0), + t.get("competition_score", 0), t.get("freshness_score", 0), + t.get("evergreen_score", 0), t["composite_score"], json.dumps(t.get("sources", [])))) + except Exception as e: + log.warning(f"Failed to insert topic {t['title']}: {e}") + db.commit() + db.close() + + log.info(f"Discovered {len(topics)} topics, stored top 25") + return topics + + +def _gather_web_topics() -> list[str]: + """Scrape trending topics from web sources.""" + topics = set() + headers = {"User-Agent": "AutoPublisher/1.0"} + + # Hacker News top stories + try: + r = requests.get(TREND_SOURCES[0]["url"], timeout=10) + if r.status_code == 200: + story_ids = r.json()[:20] + for sid in story_ids[:10]: + try: + sr = requests.get( + f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", + timeout=5) + if sr.status_code == 200: + title = sr.json().get("title", "") + if title and len(title) > 10: + topics.add(title) + except Exception: + pass + except Exception as e: + log.warning(f"HN fetch failed: {e}") + + # Reddit r/programming + try: + r = requests.get(TREND_SOURCES[1]["url"], headers=headers, timeout=10) + if r.status_code == 200: + for post in r.json()["data"]["children"][:15]: + title = post["data"].get("title", "") + if title: + topics.add(title) + except Exception as e: + log.warning(f"Reddit fetch failed: {e}") + + # GitHub trending (last 7 days) + try: + since = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") + url = TREND_SOURCES[2]["url"].format(since) + r = requests.get(url, headers=headers, timeout=10) + if r.status_code == 200: + for repo in r.json().get("items", [])[:10]: + desc = repo.get("description", "") + name = repo.get("full_name", "") + if desc: + topics.add(f"{name}: {desc}") + except Exception as e: + log.warning(f"GitHub fetch failed: {e}") + + # arXiv AI papers + try: + import xml.etree.ElementTree as ET + r = requests.get(TREND_SOURCES[3]["url"], timeout=15) + if r.status_code == 200: + root = ET.fromstring(r.text) + ns = {"atom": "http://www.w3.org/2005/Atom"} + for entry in root.findall("atom:entry", ns)[:10]: + title = entry.find("atom:title", ns) + if title is not None and title.text: + topics.add(title.text.strip().replace("\n", " ")) + except Exception as e: + log.warning(f"arXiv fetch failed: {e}") + + return list(topics) + + +def _generate_seasonal_topics() -> list[str]: + """Use LLM to generate seasonal and evergreen topic suggestions.""" + now = datetime.now() + month = now.strftime("%B") + prompt = f"""Generate 30 evergreen and seasonal content topics for {month} {now.year}. +These should be useful, educational articles people are searching for right now. +Cover these verticals: AI/ML, general tech, science, cryptocurrency, Linux, gaming, DIY/maker, and practical guides. +Respond with a JSON array of strings, each a compelling article title.""" + + try: + result = ollama_json(prompt, model="qwen3.5:4b", temperature=0.8) + if isinstance(result, list): + return result + return list(result.values())[0] if result else [] + except Exception as e: + log.warning(f"Seasonal topic generation failed: {e}") + return [] + + +def _score_and_assign(raw_topics: list[str]) -> list[dict]: + """Score topics and assign to verticals using LLM.""" + if not raw_topics: + return [] + + # Deduplicate first + unique = list(dict.fromkeys(raw_topics))[:50] + + prompt = f"""You are a content strategist. Score and categorize these {len(unique)} topics. + +Topics: +{json.dumps(unique)} + +For each topic, return: +- "title": cleaned title +- "vertical": one of (ai, tech, science, crypto, linux, gaming, diy, guides) +- "trend_score": 0-100 (how hot right now) +- "search_volume": estimated monthly searches +- "competition_score": 0-100 (how many competing articles exist) +- "freshness_score": 0-100 (how new/urgent) +- "evergreen_score": 0-100 (will this be relevant in 5 years) +- "composite_score": overall value score 0-100 (higher = publish now) + +Vertical assignment rules: +- AI/ML topics โ†’ ai +- General software/dev/cloud โ†’ tech +- Physics/biology/chemistry/space โ†’ science +- Crypto/blockchain/web3 โ†’ crypto +- Linux/FOSS/CLI/sysadmin โ†’ linux +- Games/esports/engines โ†’ gaming +- Making/building/electronics โ†’ diy +- How-to/tutorial/learning โ†’ guides + +Respond with a JSON array of objects. No markdown, no explanation.""" + + try: + result = ollama_json(prompt, model="qwen3.5:4b", temperature=0.3) + if isinstance(result, list): + return result + return [] + except Exception as e: + log.warning(f"Topic scoring failed: {e}") + return [] + + +def _deduplicate_and_rank(scored: list[dict]) -> list[dict]: + """Remove near-duplicates and rank by composite score.""" + seen_titles = set() + deduped = [] + for t in sorted(scored, key=lambda x: x.get("composite_score", 0), reverse=True): + title_lower = t["title"].lower().strip() + # Check for near-duplicates + is_dup = False + for seen in seen_titles: + if title_lower in seen or seen in title_lower: + is_dup = True + break + if not is_dup: + seen_titles.add(title_lower) + deduped.append(t) + return deduped + + +# โ”€โ”€โ”€ Research โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def research_topic(topic_id: int, topic_title: str, vertical: str) -> dict: + """Research a topic and build a knowledge package.""" + log.info(f"Researching topic #{topic_id}: {topic_title}") + + # Phase 1: Web search for sources + sources = _web_search_sources(topic_title) + + # Phase 2: LLM deep research using ornith on GamingPC + research_prompt = f"""You are an expert researcher. Deeply research this topic: + +TOPIC: {topic_title} +VERTICAL: {vertical} + +SOURCES FOUND: +{json.dumps(sources[:5], indent=2)} + +Extract and return as JSON: +{{ + "facts": ["key fact 1", "key fact 2", ...], // 8-15 verified facts + "stats": [{{"stat": "...", "source": "..."}}, ...], // 3-8 statistics with sources + "definitions": [{{"term": "...", "definition": "..."}}, ...], // key terms + "faqs": [{{"question": "...", "answer": "..."}}, ...], // 5-10 FAQs + "misconceptions": ["common wrong belief", ...], // 3-5 corrections + "timeline": [{{"date": "...", "event": "..."}}, ...], // if applicable + "citations": [{{"text": "...", "source": "..."}}, ...], // sources + "examples": ["practical example 1", ...], // 3-5 real examples + "related_concepts": ["related topic 1", ...], // 5-10 related topics + "expertise_level": "beginner|intermediate|advanced", + "summary": "one paragraph comprehensive summary" +}} + +Be accurate. Cite real sources. No hallucinations. Respond with ONLY valid JSON.""" + + try: + result = ollama_json(research_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC, + system="You are an expert research analyst. You produce accurate, well-cited research. Never fabricate information.") + except Exception as e: + log.error(f"Research LLM failed: {e}. Falling back to MacBook.") + result = ollama_json(research_prompt, model="qwen3.5:4b", + system="You are an expert research analyst. Be accurate and honest.") + + # Store knowledge package + db = init_db() + db.execute(""" + INSERT OR REPLACE INTO knowledge_packages (topic_id, facts, stats, definitions, faqs, + misconceptions, timeline, citations, examples, related_concepts, raw_sources) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + topic_id, + json.dumps(result.get("facts", [])), + json.dumps(result.get("stats", [])), + json.dumps(result.get("definitions", [])), + json.dumps(result.get("faqs", [])), + json.dumps(result.get("misconceptions", [])), + json.dumps(result.get("timeline", [])), + json.dumps(result.get("citations", [])), + json.dumps(result.get("examples", [])), + json.dumps(result.get("related_concepts", [])), + json.dumps(sources), + )) + db.execute("UPDATE topics SET knowledge_package_id = ?, status = 'researched' WHERE id = ?", + (db.execute("SELECT last_insert_rowid()").fetchone()[0], topic_id)) + db.commit() + db.close() + + log.info(f"Research complete for topic #{topic_id}") + return result + + +def _web_search_sources(topic: str) -> list[dict]: + """Search the web for topic sources.""" + sources = [] + # Try SearXNG first (self-hosted) + try: + r = requests.get("http://10.30.20.89:8888/search", + params={"q": topic, "format": "json", "categories": "general"}, + timeout=10) + if r.status_code == 200: + for result in r.json().get("results", [])[:8]: + sources.append({ + "title": result.get("title", ""), + "url": result.get("url", ""), + "snippet": result.get("content", "")[:200], + }) + except Exception: + pass + + return sources + + +# โ”€โ”€โ”€ Writing Pipeline โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def write_article(topic_id: int, topic_title: str, vertical: str, + knowledge_package: dict) -> dict: + """Multi-agent writing pipeline: outline โ†’ draft โ†’ SEO โ†’ edit โ†’ fact-check.""" + log.info(f"Writing article for topic #{topic_id}: {topic_title}") + + kp_json = json.dumps(knowledge_package, indent=2) + + # Agent 1: Outline + outline_prompt = f"""Create a detailed article outline for: + +TITLE: {topic_title} +VERTICAL: {vertical} + +KNOWLEDGE PACKAGE: +{kp_json} + +Generate an outline with: +- Introduction hook +- 5-8 major sections with subsections +- Key takeaways +- FAQ section topics +- Call-to-action + +Respond with JSON: +{{"sections": [{{"heading": "...", "subsections": ["..."]}}, ...], "faq_questions": ["..."], "cta": "..."}}""" + + outline = ollama_json(outline_prompt, model="qwen3.5:4b", temperature=0.5) + + # Agent 2: Technical Writer (ornith for quality) + draft_prompt = f"""Write a comprehensive, authoritative article. + +TITLE: {topic_title} +VERTICAL: {vertical} +OUTLINE: {json.dumps(outline)} +FACTS: {json.dumps(knowledge_package.get('facts', []))} +STATS: {json.dumps(knowledge_package.get('stats', []))} +DEFINITIONS: {json.dumps(knowledge_package.get('definitions', []))} +EXAMPLES: {json.dumps(knowledge_package.get('examples', []))} +CITATIONS: {json.dumps(knowledge_package.get('citations', []))} + +Write the full article in clean Markdown. Include: +- Engaging introduction that hooks the reader +- Well-structured sections following the outline +- Code blocks where relevant (for tech/linux) +- Pull quotes from key stats +- "Key Takeaway" boxes (use > blockquotes) +- FAQ section at the end +- Sources/citations section + +Target: 1500-3000 words. Use a clear, authoritative but conversational tone. +DO NOT use AI clichรฉs ("delve", "unleash", "game-changer", "in today's world"). +Write like an expert explaining to an intelligent peer. + +Respond with the FULL Markdown article. No JSON wrapper.""" + + draft = ollama_chat(draft_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC, + system="You are an expert technical writer. Write clear, accurate, engaging content. No AI clichรฉs. No fluff.", + temperature=0.7, max_tokens=8192) + + # Agent 3: Copy Editor (qwen, fast) + edit_prompt = f"""Edit and improve this article. Fix: +- Grammar and spelling +- Awkward phrasing +- Repetition +- Clarity issues +- Add transitions between sections +- Ensure consistent tone +- Break up overly long paragraphs + +ARTICLE: +{draft} + +Return the edited article in full Markdown. No JSON wrapper.""" + + edited = ollama_chat(edit_prompt, model="qwen3.5:4b", temperature=0.3, max_tokens=8192) + + # Agent 4: SEO Optimization + seo_prompt = f"""Optimize this article for SEO. Generate: + +1. SEO title (55-65 chars, include primary keyword) +2. Meta description (150-160 chars, compelling) +3. Suggested internal links (related topics from same vertical) +4. Tags/keywords (5-10) + +ARTICLE TITLE: {topic_title} +VERTICAL: {vertical} +FIRST 500 CHARS: {edited[:500]} + +Respond with JSON: +{{"seo_title": "...", "seo_description": "...", "keywords": ["..."], "internal_links": [{{"text": "...", "slug": "..."}}]}}""" + + seo = ollama_json(seo_prompt, model="qwen3.5:4b", temperature=0.3) + + # Agent 5: Fact Check (ornith) + factcheck_prompt = f"""Fact check this article. Verify: +1. Are the statistics accurate and properly sourced? +2. Are any claims unsubstantiated? +3. Are technical details correct? +4. Are dates and timelines accurate? +5. Is anything overstated or misleading? + +ARTICLE: +{edited[:4000]} + +FACTS USED: +{json.dumps(knowledge_package.get('facts', []))} + +Respond with JSON: +{{"passed": true/false, "issues": ["issue 1", ...], "corrections": [{{"original": "...", "corrected": "..."}}]}}""" + + factcheck = ollama_json(factcheck_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC, + system="You are a strict fact-checker. Flag everything questionable. Be conservative โ€” if unsure, flag it.", + temperature=0.1) + + # If fact check found issues, apply corrections + if not factcheck.get("passed", True): + corrections = factcheck.get("corrections", []) + if corrections: + fix_prompt = f"""Apply these corrections to the article: + +{json.dumps(corrections, indent=2)} + +ARTICLE: +{edited} + +Return the corrected article in full Markdown. No JSON wrapper.""" + edited = ollama_chat(fix_prompt, model="qwen3.5:4b", temperature=0.2) + + # Calculate stats + word_count = len(edited.split()) + reading_time = max(1, word_count // 200) + slug = topic_title.lower().strip()[:80] + slug = "".join(c if c.isalnum() or c in "- " else "" for c in slug) + slug = slug.replace(" ", "-").strip("-") + + # Store article + db = init_db() + db.execute(""" + INSERT OR REPLACE INTO articles (topic_id, vertical, title, slug, content_md, + seo_title, seo_description, word_count, reading_time_minutes, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft') + """, (topic_id, vertical, topic_title, slug, edited, + seo.get("seo_title", topic_title[:65]), + seo.get("seo_description", ""), + word_count, reading_time)) + db.execute("UPDATE topics SET status = 'written', article_id = last_insert_rowid() WHERE id = ?", + (topic_id,)) + db.commit() + db.close() + + log.info(f"Article written for #{topic_id}: {word_count} words, {reading_time}min read") + return { + "topic_id": topic_id, + "title": topic_title, + "slug": slug, + "content": edited, + "seo": seo, + "word_count": word_count, + "reading_time": reading_time, + "factcheck_passed": factcheck.get("passed", True), + } + + +# โ”€โ”€โ”€ Site Builder โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def build_site(vertical: str, articles: list[dict]) -> str: + """Build static HTML site from articles.""" + log.info(f"Building site for {vertical}...") + + site_dir = BASE_DIR / "sites" / vertical + site_dir.mkdir(parents=True, exist_ok=True) + (site_dir / "articles").mkdir(exist_ok=True) + (site_dir / "assets" / "images").mkdir(parents=True, exist_ok=True) + + # Generate each article page + for article in articles: + html = _article_to_html(article, vertical) + article_path = site_dir / "articles" / f"{article['slug']}.html" + article_path.write_text(html) + + # Generate homepage + homepage = _build_homepage(vertical, articles) + (site_dir / "index.html").write_text(homepage) + + # Generate RSS feed + rss = _build_rss(vertical, articles) + (site_dir / "rss.xml").write_text(rss) + + # Generate sitemap + sitemap = _build_sitemap(vertical, articles) + (site_dir / "sitemap.xml").write_text(sitemap) + + log.info(f"Site built for {vertical}: {len(articles)} articles") + return str(site_dir) + + +def _article_to_html(article: dict, vertical: str) -> str: + """Convert markdown article to full HTML page.""" + # Simple markdown-to-HTML conversion (rudimentary but functional) + content = article.get("content_md", article.get("content", "")) + html_body = _md_to_html(content) + + seo_title = article.get("seo_title", article.get("title", "")) + seo_desc = article.get("seo_description", "") + og_image = article.get("og_image", f"/assets/images/{vertical}-default.webp") + json_ld = _build_json_ld(article, vertical) + + return f""" + + + + + {seo_title} + + + + + + + + + + + + +
+ +
+
+
+
+

{article.get("title", "")}

+
+ + {article.get("reading_time_minutes", 5)} min read + {article.get("word_count", 0)} words +
+
+
+ {html_body} +
+
+
+ {_build_tag_links(article.get("keywords", []), vertical)} +
+
+

Sources

+ {_build_sources_html(article)} +
+
+
+ +
+ + +""" + + +def _md_to_html(md: str) -> str: + """Basic markdown to HTML conversion.""" + import re + lines = md.split("\n") + html = [] + in_code_block = False + code_lines = [] + code_lang = "" + + i = 0 + while i < len(lines): + line = lines[i] + + # Code blocks + if line.strip().startswith("```"): + if in_code_block: + code = "\n".join(code_lines) + html.append(f'
{_escape_html(code)}
') + code_lines = [] + in_code_block = False + else: + in_code_block = True + code_lang = line.strip()[3:].strip() + i += 1 + continue + + if in_code_block: + code_lines.append(line) + i += 1 + continue + + # Headers + if line.startswith("### "): + html.append(f"

{_inline_md(line[4:])}

") + elif line.startswith("## "): + html.append(f"

{_inline_md(line[3:])}

") + elif line.startswith("# "): + html.append(f"

{_inline_md(line[2:])}

") + + # Blockquotes + elif line.startswith("> "): + html.append(f'

{_inline_md(line[2:])}

') + + # Lists + elif line.strip().startswith("- ") or line.strip().startswith("* "): + html.append(f"
  • {_inline_md(line.strip()[2:])}
    • ") + elif re.match(r"^\d+\.", line.strip()): + text = re.sub(r"^\d+\.\s*", "", line.strip()) + html.append(f"
  • {_inline_md(text)}
    • ") + + # Horizontal rule + elif line.strip() in ("---", "***", "___"): + html.append("
    ") + + # Empty line + elif not line.strip(): + html.append("") + + # Paragraph + else: + html.append(f"

    {_inline_md(line)}

    ") + + i += 1 + + return "\n".join(html) + + +def _inline_md(text: str) -> str: + """Convert inline markdown to HTML.""" + import re + # Bold + text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) + # Italic + text = re.sub(r"\*(.+?)\*", r"\1", text) + # Inline code + text = re.sub(r"`(.+?)`", r"\1", text) + # Links + text = re.sub(r"\[(.+?)\]\((.+?)\)", r'\1', text) + return text + + +def _escape_html(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _build_homepage(vertical: str, articles: list[dict]) -> str: + """Build the homepage for a vertical site.""" + articles_html = "" + for a in sorted(articles, key=lambda x: x.get("published_at", ""), reverse=True)[:20]: + articles_html += f""" +
    +

    {a.get('title', '')}

    +

    {a.get('published_at', '')} ยท {a.get('reading_time_minutes', 5)} min read

    +

    {a.get('seo_description', '')}

    +
    """ + + return f""" + + + + + {vertical}.thetempleofdoom.com โ€” {vertical.title()} articles, guides & insights + + + + + + +
    + +
    +
    +
    +

    {vertical.title()} Insights & Guides

    +

    Expert articles, tutorials, and deep dives. Updated daily.

    +
    +
    + {articles_html} +
    +
    + + +""" + + +def _build_rss(vertical: str, articles: list[dict]) -> str: + """Build RSS 2.0 feed.""" + items = "" + domain = f"{vertical}.thetempleofdoom.com" + for a in sorted(articles, key=lambda x: x.get("published_at", ""), reverse=True)[:20]: + items += f""" + + {_escape_xml(a.get('title', ''))} + https://{domain}/articles/{a.get('slug', '')}.html + https://{domain}/articles/{a.get('slug', '')}.html + {_escape_xml(a.get('seo_description', ''))} + {a.get('published_at', '')} + """ + + return f""" + + + {vertical}.thetempleofdoom.com + https://{domain} + Expert {vertical} articles and insights + en-us + {datetime.now().isoformat()} + + {items} + +""" + + +def _build_sitemap(vertical: str, articles: list[dict]) -> str: + """Build XML sitemap.""" + urls = "" + domain = f"{vertical}.thetempleofdoom.com" + urls += f""" + + https://{domain}/ + daily + 1.0 + """ + for a in articles: + urls += f""" + + https://{domain}/articles/{a.get('slug', '')}.html + {a.get('published_at', datetime.now().strftime('%Y-%m-%d'))} + weekly + 0.8 + """ + return f""" +{urls} +""" + + +def _build_json_ld(article: dict, vertical: str) -> dict: + return { + "@context": "https://schema.org", + "@type": "Article", + "headline": article.get("title", ""), + "description": article.get("seo_description", ""), + "datePublished": article.get("published_at", ""), + "author": {"@type": "Organization", "name": f"{vertical}.thetempleofdoom.com"}, + "publisher": {"@type": "Organization", "name": f"{vertical}.thetempleofdoom.com"}, + } + + +def _site_json_ld(vertical: str) -> dict: + return { + "@context": "https://schema.org", + "@type": "WebSite", + "name": f"{vertical}.thetempleofdoom.com", + "url": f"https://{vertical}.thetempleofdoom.com", + "description": f"Expert {vertical} articles, tutorials, and insights. Updated daily.", + } + + +def _build_tag_links(keywords: list, vertical: str) -> str: + return " ".join(f'{k}' for k in (keywords or [])) + + +def _build_sources_html(article: dict) -> str: + # Extract sources from knowledge package if available + return "

    Sources available in the original knowledge package.

    " + + +def _escape_xml(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + + +# โ”€โ”€โ”€ Deploy โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def deploy_site(vertical: str, site_dir: str, ct_ip: str) -> bool: + """Deploy a site to its Proxmox CT.""" + log.info(f"Deploying {vertical} to {ct_ip}...") + + # Create deployment tarball + import tarfile + tarball = BASE_DIR / "core" / f"{vertical}_deploy.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + tar.add(site_dir, arcname=vertical) + + try: + # Push to CT + result = os.system(f"scp {tarball} root@{ct_ip}:/tmp/ 2>/dev/null") + if result != 0: + log.error(f"scp failed for {vertical}") + return False + + os.system(f"ssh root@{ct_ip} 'cd /tmp && tar xzf {vertical}_deploy.tar.gz && " + f"cp -r {vertical}/* /var/www/html/ && systemctl restart nginx' 2>/dev/null") + + # Verify + time.sleep(2) + verify = os.popen(f"curl -s -o /dev/null -w '%{{http_code}}' http://{ct_ip}:80/").read().strip() + if verify == "200": + log.info(f"Deploy {vertical} successful: HTTP {verify}") + return True + else: + log.error(f"Deploy {vertical} verify failed: HTTP {verify}") + return False + except Exception as e: + log.error(f"Deploy {vertical} exception: {e}") + return False + finally: + if tarball.exists(): + tarball.unlink() + + +# โ”€โ”€โ”€ Pipeline Runner โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def run_daily_pipeline(max_articles: int = 3): + """Run the full daily publishing pipeline.""" + log.info("=" * 60) + log.info("DAILY PUBLISHING PIPELINE STARTING") + log.info("=" * 60) + + run_id = None + db = init_db() + try: + # Log the run + cur = db.execute("INSERT INTO pipeline_runs (run_type, started_at) VALUES ('daily', datetime('now'))") + run_id = cur.lastrowid + db.commit() + + # Step 1: Discover trends + topics = discover_trends() + db.execute("UPDATE pipeline_runs SET topics_discovered = ? WHERE id = ?", + (len(topics), run_id)) + db.commit() + + if not topics: + log.warning("No topics discovered. Exiting.") + return + + # Step 2: Get approved topics (composite_score > 60, not yet published) + top_topics = db.execute(""" + SELECT id, title, vertical, composite_score FROM topics + WHERE status = 'discovered' AND composite_score > 60 + ORDER BY composite_score DESC LIMIT ? + """, (max_articles,)).fetchall() + + articles_published = 0 + vertical_articles = {} + + for topic_row in top_topics: + topic_id, topic_title, vertical, score = topic_row + + # Step 3: Research + kp = research_topic(topic_id, topic_title, vertical) + + # Step 4: Write + article = write_article(topic_id, topic_title, vertical, kp) + + # Collect articles per vertical for site building + if vertical not in vertical_articles: + vertical_articles[vertical] = [] + vertical_articles[vertical].append(article) + + articles_published += 1 + log.info(f" โœ“ Published: {topic_title} โ†’ {vertical}") + + # Step 5: Build and deploy each vertical site + for vertical, articles in vertical_articles.items(): + site_dir = build_site(vertical, articles) + vinfo = VERTICALS.get(vertical, {}) + ct_ip = vinfo.get("ip") + if ct_ip: + deploy_site(vertical, site_dir, ct_ip) + else: + log.warning(f"No CT IP for {vertical} โ€” skipping deploy") + + # Update run log + db.execute(""" + UPDATE pipeline_runs SET articles_published = ?, finished_at = datetime('now') + WHERE id = ? + """, (articles_published, run_id)) + db.commit() + + log.info(f"Pipeline complete: {articles_published} articles published across {len(vertical_articles)} sites") + + except Exception as e: + log.error(f"Pipeline failed: {e}", exc_info=True) + if run_id: + db.execute("UPDATE pipeline_runs SET errors = ?, finished_at = datetime('now') WHERE id = ?", + (json.dumps([str(e)]), run_id)) + db.commit() + finally: + db.close() + + +# โ”€โ”€โ”€ CLI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser(description="Autonomous Publishing System") + ap.add_argument("command", choices=["init", "discover", "research", "write", "build", "deploy", "run", "status"]) + ap.add_argument("--topic-id", type=int, help="Topic ID for research/write") + ap.add_argument("--vertical", type=str, help="Vertical name") + ap.add_argument("--max", type=int, default=3, help="Max articles per run") + + args = ap.parse_args() + + if args.command == "init": + init_db() + print("โœ“ Database initialized") + + elif args.command == "discover": + topics = discover_trends() + for t in topics[:15]: + print(f" [{t['vertical']}] {t['title'][:80]} (score: {t['composite_score']})") + + elif args.command == "run": + run_daily_pipeline(max_articles=args.max) + + elif args.command == "status": + db = init_db() + run = db.execute("SELECT * FROM pipeline_runs ORDER BY id DESC LIMIT 1").fetchone() + topics_count = db.execute("SELECT COUNT(*) FROM topics").fetchone()[0] + articles_count = db.execute("SELECT COUNT(*) FROM articles").fetchone()[0] + published = db.execute("SELECT COUNT(*) FROM articles WHERE status = 'published'").fetchone()[0] + print(f"Last run: {run}") + print(f"Topics in DB: {topics_count}") + print(f"Articles: {articles_count} ({published} published)") + + else: + print(f"Unknown command: {args.command}") diff --git a/core/publisher.db b/core/publisher.db new file mode 100644 index 0000000000000000000000000000000000000000..5f51ff2d237f5d4cba0bfcee55188541e5f3cff6 GIT binary patch literal 49152 zcmeI&TWjM+6bEoC7dy@c_Nitc!s?}D!!F%=mpm5Yrs@*vI9bQ04NDmm>DX2yONwTc zY@R~`-O_K+ue9{B^qUk4r6bw0Grp|62^4Dn1}C;=G?sqn%oz#$_)XOjq__OAYYP2U z?om$Da=W^o%jNR&xh|jSBYwY>zLEd6JFlm`%;&!N^{H&*+YI96ITK04daKA zCgBH;#hLAJ%fFzIhGTzYd|PQ&8~T^(SY{|3%O%d*EibB}by{JTTk7J}eeW zJ3HEKr57~{lUd+d#9T5%&+*#9gl(&83@7w>a42Prq`&F*aGJIKH%&u5H5pMBm_ktK zWfFGCw3Bq~P4dkCejuE#>9R{dw3+2co=8(sJ%&@_g2}FwIpP^zC4)I0ciJ71**^;G zIOmkOO%M7oqOVu;rMEj;&hcz|$Io3^*_as#KY7n41(TI0ZC$(GT`82FJkf3slhiOh z)4h@r;e!`NB?$(b9~}Aq#ACzhubL~Fp*96(n=Tx>x1K$RN6by(DMbbFOW{X|Z_jPsOFm2Eb0D|BRS@VyCx`)wwRhRlxf)#oQJGsm5D zMB^1^T3&^+hRJ%|OT@0@MM74cjKq~ohloaNjUAdTF|x)CmMo>#nUUegmt!ukcdZV)@ZG3OS#{O& zy8+3SLgwQPL$^ZW9gjGlp)bjEYKKI#j2jz1Rwwb2tWV=5P$##AOL}R z6 + + + + + Auto Publisher โ€” Admin Dashboard + + + +

    ๐Ÿค– Auto Publisher

    +

    Autonomous Publishing System โ€” Admin Dashboard

    + + +
    +
    +
    Topics Discovered
    +
    {{ stats.topics }}
    +
    +
    +
    Articles Written
    +
    {{ stats.articles }}
    +
    +
    +
    Published
    +
    {{ stats.published }}
    +
    +
    +
    Last Run
    +
    {{ stats.last_run or 'Never' }}
    +
    +
    +
    Total Pageviews
    +
    {{ stats.total_views }}
    +
    +
    +
    Avg Score
    +
    {{ stats.avg_score }}
    +
    +
    + + +
    +

    ๐Ÿ“ก Sites

    +
    + {% for v in verticals %} +
    +
    {{ v }}.thetempleofdoom.com
    +
    + {{ 'โ— Live' if v in live_sites else 'โ—‹ Pending' }} +
    +
    + {% endfor %} +
    +
    + + +
    +

    ๐ŸŽฎ Controls

    +
    +
    + +
    +
    + +
    + +
    +
    + + +
    +

    ๐Ÿ“‹ Recent Topics

    + + + + + + {% for t in topics %} + + + + + + + + {% endfor %} + +
    TitleVerticalScoreStatusDiscovered
    {{ t.title[:80] }}{{ t.vertical }}{{ t.composite_score }}{{ t.status }}{{ t.created_at[:10] if t.created_at else '-' }}
    +
    + + +
    +

    ๐Ÿ“ Recent Articles

    + + + + + + {% for a in articles %} + + + + + + + + {% endfor %} + +
    TitleVerticalWordsStatusCreated
    {{ a.title[:80] if a.title else 'Untitled' }}{{ a.vertical }}{{ a.word_count }}{{ a.status }}{{ a.created_at[:10] if a.created_at else '-' }}
    +
    + + +
    +

    ๐Ÿ“œ Pipeline Log

    +
    + {% for entry in log_lines %} +
    [{{ entry.time }}] {{ entry.msg }}
    + {% endfor %} +
    +
    + +""" + + +@app.route("/") +def index(): + db = get_db() + stats = { + "topics": db.execute("SELECT COUNT(*) FROM topics").fetchone()[0], + "articles": db.execute("SELECT COUNT(*) FROM articles").fetchone()[0], + "published": db.execute("SELECT COUNT(*) FROM articles WHERE status='published'").fetchone()[0], + "last_run": None, + "total_views": db.execute("SELECT COALESCE(SUM(pageviews), 0) FROM analytics").fetchone()[0], + "avg_score": round(db.execute("SELECT COALESCE(AVG(composite_score), 0) FROM topics WHERE composite_score > 0").fetchone()[0], 1), + } + last_run = db.execute("SELECT started_at FROM pipeline_runs ORDER BY id DESC LIMIT 1").fetchone() + if last_run: + stats["last_run"] = last_run[0] + + topics = db.execute("SELECT * FROM topics ORDER BY created_at DESC LIMIT 20").fetchall() + articles = db.execute("SELECT * FROM articles ORDER BY created_at DESC LIMIT 20").fetchall() + + # Read log + log_path = BASE_DIR / "core" / "orchestrator.log" + log_lines = [] + if log_path.exists(): + for line in log_path.read_text().split("\n")[-30:]: + if not line.strip(): + continue + level = "" + if "ERROR" in line: + level = "error" + elif "โœ“" in line or "successful" in line.lower(): + level = "success" + log_lines.append({ + "time": line[:19] if len(line) > 19 else "", + "msg": line, + "level": level, + }) + + live_sites = ["ai", "tech"] # Will be dynamically checked + + return render_template_string( + DASHBOARD_HTML, + stats=stats, + topics=topics, + articles=articles, + log_lines=log_lines, + verticals=VERTICALS, + live_sites=live_sites, + ) + + +@app.route("/api/run", methods=["POST"]) +def api_run(): + """Trigger a pipeline run.""" + from orchestrator import run_daily_pipeline + import threading + def _run(): + run_daily_pipeline(max_articles=3) + t = threading.Thread(target=_run, daemon=True) + t.start() + return jsonify({"status": "started", "message": "Pipeline running in background"}) + + +@app.route("/api/discover", methods=["POST"]) +def api_discover(): + """Trigger trend discovery.""" + from orchestrator import discover_trends + import threading + def _run(): + discover_trends() + t = threading.Thread(target=_run, daemon=True) + t.start() + return jsonify({"status": "started", "message": "Trend discovery running"}) + + +@app.route("/api/stats") +def api_stats(): + db = get_db() + return jsonify({ + "topics_total": db.execute("SELECT COUNT(*) FROM topics").fetchone()[0], + "articles_total": db.execute("SELECT COUNT(*) FROM articles").fetchone()[0], + "published": db.execute("SELECT COUNT(*) FROM articles WHERE status='published'").fetchone()[0], + "by_vertical": { + v: db.execute("SELECT COUNT(*) FROM topics WHERE vertical=?", (v,)).fetchone()[0] + for v in VERTICALS + }, + "pipeline_runs": db.execute("SELECT COUNT(*) FROM pipeline_runs").fetchone()[0], + }) + + +@app.route("/health") +def health(): + return jsonify({"status": "healthy", "timestamp": datetime.now().isoformat()}) + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=5106) + ap.add_argument("--host", type=str, default="127.0.0.1") + args = ap.parse_args() + print(f"Admin Dashboard โ†’ http://{args.host}:{args.port}") + app.run(host=args.host, port=args.port, debug=False) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..064a58e --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,270 @@ +# Autonomous AI Publishing System โ€” Architecture + +## Overview + +A fully autonomous, self-hosted publishing platform that continuously discovers valuable topics, generates original useful content, and publishes to a collection of evergreen authority websites. + +## Sites (Vertical Authority Domains) + +| Site | Domain | CT ID | IP | Port | +|------|--------|-------|-----|------| +| AI | ai.thetempleofdoom.com | TBD | TBD | 5000 | +| Tech | tech.thetempleofdoom.com | TBD | TBD | 5000 | +| Science | science.thetempleofdoom.com | TBD | TBD | 5000 | +| Crypto | crypto.thetempleofdoom.com | TBD | TBD | 5000 | +| Linux | linux.thetempleofdoom.com | TBD | TBD | 5000 | +| Gaming | gaming.thetempleofdoom.com | TBD | TBD | 5000 | +| DIY | diy.thetempleofdoom.com | TBD | TBD | 5000 | +| Guides | guides.thetempleofdoom.com | TBD | TBD | 5000 | + +## System Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ CRON ORCHESTRATOR (MacBook) โ”‚ +โ”‚ Every morning @ 6AM Pacific โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ TREND DISCOVERY ENGINE โ”‚ +โ”‚ Sources: News APIs, RSS, Reddit, HN, GitHub, Google Trends, โ”‚ +โ”‚ arXiv, Stack Overflow, Twitter/X, seasonal calendar โ”‚ +โ”‚ Output: Scored topic list with vertical assignments โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ RESEARCH AGENT โ”‚ +โ”‚ Per topic: multi-source extraction โ†’ facts, stats, citations, โ”‚ +โ”‚ FAQs, timelines, misconceptions, examples โ”‚ +โ”‚ LLM: GamingPC ornith:latest for deep research โ”‚ +โ”‚ Output: Knowledge Package (JSON) โ†’ stored in SQLite โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ MULTI-AGENT WRITING PIPELINE โ”‚ +โ”‚ Outline Agent โ†’ Technical Writer โ†’ SEO Writer โ†’ Copy Editor โ”‚ +โ”‚ โ†’ Fact Checker โ†’ Quality Reviewer โ”‚ +โ”‚ LLMs: qwen3.5:4b (fast gate) + ornith:latest (verify) โ”‚ +โ”‚ Output: Polished Markdown article + frontmatter โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ IMAGE GENERATION โ”‚ +โ”‚ FLUX via FAL.ai for hero images, diagrams, charts โ”‚ +โ”‚ Output: WebP images in site assets/ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ SEO PROCESSOR โ”‚ +โ”‚ Meta titles, descriptions, OG tags, JSON-LD, schema.org, โ”‚ +โ”‚ canonical URLs, breadcrumbs, XML sitemap, robots.txt โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ SITE BUILDER (per vertical) โ”‚ +โ”‚ Static site generator: Markdown โ†’ HTML, rebuilds on new content โ”‚ +โ”‚ Updates: homepage, category pages, RSS, sitemap, related links โ”‚ +โ”‚ Built-in search via pagefind โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ DEPLOYMENT โ”‚ +โ”‚ 1. Build static site locally โ”‚ +โ”‚ 2. scp to Proxmox CT โ”‚ +โ”‚ 3. Restart nginx/service โ”‚ +โ”‚ 4. Ping sitemap to Google/Bing โ”‚ +โ”‚ 5. Verify live โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ ANALYTICS + LEARNING LOOP โ”‚ +โ”‚ Nightly: analyze pageviews, time-on-page, bounce rate โ”‚ +โ”‚ Feed back into topic scoring โ†’ improve selection over time โ”‚ +โ”‚ Plausible-style privacy-first analytics, self-hosted โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Service APIs (REST) + +Every component is an independent API service: + +### Trend Discovery API (port 5100) +- `POST /api/trends/discover` โ€” Run full discovery scan +- `GET /api/trends/scored` โ€” Get scored topic list +- `GET /api/trends/sources` โ€” List active data sources + +### Research API (port 5101) +- `POST /api/research/topic` โ€” Research a topic, return knowledge package +- `GET /api/research/package/{id}` โ€” Get stored knowledge package +- `GET /api/research/sources/{topic}` โ€” Raw sources for a topic + +### Writing API (port 5102) +- `POST /api/write/article` โ€” Generate article from knowledge package +- `GET /api/write/draft/{id}` โ€” Get draft +- `POST /api/write/review/{id}` โ€” Request quality review + +### SEO API (port 5103) +- `POST /api/seo/optimize` โ€” SEO-optimize an article +- `POST /api/seo/sitemap` โ€” Generate sitemap for a site +- `POST /api/seo/structured-data` โ€” Generate JSON-LD + +### Image API (port 5104) +- `POST /api/images/generate` โ€” Generate image for article +- `POST /api/images/optimize` โ€” Optimize existing image + +### Deploy API (port 5105) +- `POST /api/deploy/site/{name}` โ€” Build and deploy a site +- `POST /api/deploy/all` โ€” Deploy all sites +- `GET /api/deploy/status/{name}` โ€” Deployment status + +### Admin Dashboard (port 5106) +- Full control panel UI +- Topic approval, manual triggers, analytics views + +## Database Schema + +### topics +```sql +CREATE TABLE topics ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + vertical TEXT NOT NULL, -- ai, tech, science, crypto, linux, gaming, diy, guides + trend_score REAL, + search_volume INTEGER, + competition_score REAL, + freshness_score REAL, + evergreen_score REAL, + composite_score REAL, + sources TEXT, -- JSON array + status TEXT DEFAULT 'discovered', -- discovered, approved, researching, writing, reviewing, published, rejected + knowledge_package_id INTEGER, + article_id INTEGER, + published_url TEXT, + published_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +### knowledge_packages +```sql +CREATE TABLE knowledge_packages ( + id INTEGER PRIMARY KEY, + topic_id INTEGER, + facts TEXT, -- JSON + stats TEXT, -- JSON + definitions TEXT, -- JSON + faqs TEXT, -- JSON + misconceptions TEXT, -- JSON + timeline TEXT, -- JSON + citations TEXT, -- JSON + examples TEXT, -- JSON + related_concepts TEXT, -- JSON + raw_sources TEXT, -- JSON + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +### articles +```sql +CREATE TABLE articles ( + id INTEGER PRIMARY KEY, + topic_id INTEGER, + vertical TEXT, + title TEXT, + slug TEXT UNIQUE, + content_md TEXT, + content_html TEXT, + seo_title TEXT, + seo_description TEXT, + og_image TEXT, + json_ld TEXT, + word_count INTEGER, + reading_time_minutes INTEGER, + status TEXT DEFAULT 'draft', + published_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +### analytics +```sql +CREATE TABLE analytics ( + id INTEGER PRIMARY KEY, + article_id INTEGER, + vertical TEXT, + pageviews INTEGER DEFAULT 0, + unique_visitors INTEGER DEFAULT 0, + avg_time_on_page REAL, + bounce_rate REAL, + referrers TEXT, -- JSON + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +### performance_learning +```sql +CREATE TABLE performance_learning ( + id INTEGER PRIMARY KEY, + vertical TEXT, + top_performing_patterns TEXT, -- JSON + headline_formats TEXT, -- JSON + optimal_word_count INTEGER, + best_publish_times TEXT, -- JSON + keyword_insights TEXT, -- JSON + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +## LLM Strategy + +| Task | Model | Host | Reasoning | +|------|-------|------|-----------| +| Topic scoring | qwen3.5:4b | MacBook | Fast, cheap | +| Research extraction | ornith:latest | GamingPC | Deep reasoning | +| Outline generation | qwen3.5:4b | MacBook | Structural, fast | +| Technical writing | ornith:latest | GamingPC | Quality matters | +| SEO optimization | qwen3.5:4b | MacBook | Pattern matching | +| Copy editing | qwen3.5:4b | MacBook | Fast iteration | +| Fact checking | ornith:latest | GamingPC | Accuracy critical | +| Quality review | ornith:latest | GamingPC | Final gate | + +## Deployment Strategy + +Each site is a standalone Proxmox CT running: +- Python Flask/FastAPI (static site generator) +- nginx (serving static files) +- pagefind (search index) +- cloudflared (tunnel to CF) + +Core orchestrator runs on MacBook via cron (launchd-supervised). + +## Security Model + +- All APIs internal-only (10.30.20.0/24) +- Admin dashboard: localhost only, auth via Hermes +- Cloudflare tunnels: only expose nginx on :80 +- No secrets in code โ€” env vars only +- Gitea private repos for all sites + +## Backup Strategy + +- All article content in Gitea (git history = backup) +- SQLite DBs synced to iCloud daily +- Proxmox CT snapshots weekly +- Knowledge packages exported to JSON nightly + +## Monitoring + +- Health checks on all 8 site CTs +- Orchestrator pipeline status dashboard +- LLM usage/cost tracking +- Publish success/failure alerts via Telegram diff --git a/shared/assets/style.css b/shared/assets/style.css new file mode 100644 index 0000000..1840ced --- /dev/null +++ b/shared/assets/style.css @@ -0,0 +1,259 @@ +/* ========================================================================== + Autonomous Publishing System โ€” Shared Site CSS + Clean, readable, fast-loading design system for all vertical sites. + ========================================================================== */ + +:root { + --bg: #ffffff; + --bg-secondary: #f8f9fa; + --text: #1a1a2e; + --text-secondary: #555; + --primary: #2563eb; + --primary-hover: #1d4ed8; + --border: #e5e7eb; + --accent: #10b981; + --card-bg: #ffffff; + --code-bg: #1e293b; + --code-text: #e2e8f0; + --max-width: 800px; + --font-sans: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; + --font-mono: 'SF Mono', 'Fira Code', 'Fira Mono', 'Roboto Mono', monospace; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #0f172a; + --bg-secondary: #1e293b; + --text: #e2e8f0; + --text-secondary: #94a3b8; + --primary: #3b82f6; + --primary-hover: #60a5fa; + --border: #334155; + --card-bg: #1e293b; + --code-bg: #0f172a; + } +} + +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +html { font-size: 18px; line-height: 1.7; } + +body { + font-family: var(--font-sans); + background: var(--bg); + color: var(--text); + max-width: var(--max-width); + margin: 0 auto; + padding: 0 1.5rem; + -webkit-font-smoothing: antialiased; +} + +/* โ”€โ”€โ”€ Header โ”€โ”€โ”€ */ +header { + padding: 1.5rem 0; + border-bottom: 1px solid var(--border); + margin-bottom: 2rem; +} + +nav { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 1rem; +} + +.logo { + font-weight: 700; + font-size: 1.2rem; + color: var(--primary); + text-decoration: none; + letter-spacing: -0.02em; +} + +.nav-links { display: flex; gap: 1.5rem; } +.nav-links a { + color: var(--text-secondary); + text-decoration: none; + font-size: 0.9rem; + transition: color 0.2s; +} +.nav-links a:hover { color: var(--primary); } + +.rss-link { + background: var(--bg-secondary); + padding: 0.25rem 0.75rem; + border-radius: 4px; + font-weight: 500; +} + +/* โ”€โ”€โ”€ Hero โ”€โ”€โ”€ */ +.hero { + padding: 3rem 0; + text-align: center; +} +.hero h1 { font-size: 2.2rem; margin-bottom: 0.5rem; letter-spacing: -0.03em; } +.hero p { color: var(--text-secondary); font-size: 1.1rem; } + +/* โ”€โ”€โ”€ Article Cards โ”€โ”€โ”€ */ +.articles-grid { + display: flex; + flex-direction: column; + gap: 1.5rem; + margin-bottom: 3rem; +} + +.card { + background: var(--card-bg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 1.5rem; + transition: border-color 0.2s, box-shadow 0.2s; +} +.card:hover { + border-color: var(--primary); + box-shadow: 0 2px 12px rgba(37, 99, 235, 0.1); +} +.card h2 { + font-size: 1.3rem; + margin-bottom: 0.25rem; + line-height: 1.3; +} +.card h2 a { color: var(--text); text-decoration: none; } +.card h2 a:hover { color: var(--primary); } +.card .meta { + color: var(--text-secondary); + font-size: 0.85rem; + margin-bottom: 0.5rem; +} +.card p { color: var(--text-secondary); font-size: 0.95rem; } + +/* โ”€โ”€โ”€ Article Page โ”€โ”€โ”€ */ +article { margin-bottom: 3rem; } + +.article-header { + margin-bottom: 2rem; + padding-bottom: 1.5rem; + border-bottom: 1px solid var(--border); +} +.article-header h1 { + font-size: 2rem; + line-height: 1.2; + margin-bottom: 0.5rem; + letter-spacing: -0.03em; +} +.article-header .meta { + color: var(--text-secondary); + font-size: 0.9rem; + display: flex; + gap: 1rem; + flex-wrap: wrap; +} + +.article-content { margin-bottom: 2rem; } +.article-content h2 { + font-size: 1.5rem; + margin: 2.5rem 0 0.75rem; + padding-bottom: 0.25rem; + border-bottom: 2px solid var(--primary); + display: inline-block; +} +.article-content h3 { + font-size: 1.2rem; + margin: 1.5rem 0 0.5rem; +} +.article-content p { margin-bottom: 1.2rem; } +.article-content ul, .article-content ol { + margin: 0 0 1.2rem 1.5rem; +} +.article-content li { margin-bottom: 0.4rem; } +.article-content a { color: var(--primary); text-decoration: underline; } +.article-content strong { font-weight: 600; } +.article-content blockquote { + border-left: 4px solid var(--primary); + padding: 0.75rem 1.5rem; + margin: 1.5rem 0; + background: var(--bg-secondary); + border-radius: 0 6px 6px 0; + font-style: italic; +} +.article-content blockquote p { margin-bottom: 0; } + +.article-content pre { + background: var(--code-bg); + color: var(--code-text); + padding: 1.25rem; + border-radius: 8px; + overflow-x: auto; + margin: 1.5rem 0; + font-family: var(--font-mono); + font-size: 0.85rem; + line-height: 1.6; +} +.article-content code { + font-family: var(--font-mono); + font-size: 0.85em; + background: var(--bg-secondary); + padding: 0.15em 0.4em; + border-radius: 3px; +} +.article-content pre code { + background: none; + padding: 0; +} + +.article-content hr { + border: none; + border-top: 1px solid var(--border); + margin: 2rem 0; +} + +/* โ”€โ”€โ”€ Tags โ”€โ”€โ”€ */ +.tags { display: flex; gap: 0.5rem; flex-wrap: wrap; margin-bottom: 1.5rem; } +.tag { + background: var(--bg-secondary); + color: var(--text-secondary); + padding: 0.25rem 0.75rem; + border-radius: 20px; + font-size: 0.8rem; + text-decoration: none; + transition: background 0.2s, color 0.2s; +} +.tag:hover { background: var(--primary); color: white; } + +/* โ”€โ”€โ”€ Sources โ”€โ”€โ”€ */ +.sources { margin-top: 2rem; padding-top: 1rem; border-top: 1px solid var(--border); } +.sources h3 { font-size: 1rem; color: var(--text-secondary); margin-bottom: 0.5rem; } + +/* โ”€โ”€โ”€ Related โ”€โ”€โ”€ */ +.related { + margin-top: 3rem; + padding-top: 1.5rem; + border-top: 1px solid var(--border); +} + +/* โ”€โ”€โ”€ Footer โ”€โ”€โ”€ */ +.site-footer { + margin-top: 4rem; + padding: 1.5rem 0; + border-top: 1px solid var(--border); + text-align: center; + color: var(--text-secondary); + font-size: 0.85rem; + display: flex; + justify-content: space-between; + flex-wrap: wrap; + gap: 1rem; +} +.site-footer nav { display: flex; gap: 1.5rem; } +.site-footer a { color: var(--text-secondary); text-decoration: none; } +.site-footer a:hover { color: var(--primary); } + +/* โ”€โ”€โ”€ Responsive โ”€โ”€โ”€ */ +@media (max-width: 600px) { + html { font-size: 16px; } + body { padding: 0 1rem; } + .hero h1 { font-size: 1.6rem; } + .article-header h1 { font-size: 1.5rem; } + nav { flex-direction: column; align-items: flex-start; } +} diff --git a/sites/ai b/sites/ai new file mode 160000 index 0000000..55e4e01 --- /dev/null +++ b/sites/ai @@ -0,0 +1 @@ +Subproject commit 55e4e0177223fb9052d52b80816a86f909192071 diff --git a/sites/tech b/sites/tech new file mode 160000 index 0000000..9e1cc21 --- /dev/null +++ b/sites/tech @@ -0,0 +1 @@ +Subproject commit 9e1cc21d7c700d0de666cf7a42935ba971caf96c