Autonomous Publishing System — full stack: orchestrator, 8 vertical sites, admin dashboard, analytics, cron pipeline

core/cloudflare_setup.py

@@ -0,0 +1,184 @@
+"""
+Wire auto-publisher sites to Cloudflare home tunnel.
+Adds ingress rules to the existing home tunnel for each site subdomain.
+"""
+import json
+import subprocess
+import time
+
+# Home tunnel ID (the existing tunnel that routes ~38 services)
+HOME_TUNNEL_ID = "d2871458-1737-4844-b114-9a44b9d71e25"
+ACCT_ID = "895479ab3540daa6d61ae702a5164475"
+ZONE_ID = "93634d23ff8138fbd795763b681158ac"
+DOMAIN = "thetempleofdoom.com"
+
+# Site IPs
+SITES = {
+    "ai":      "10.30.20.240",
+    "tech":    "10.30.20.241",
+    "science": "10.30.20.242",
+    "crypto":  "10.30.20.243",
+    "linux":   "10.30.20.244",
+    "gaming":  "10.30.20.246",
+    "diy":     "10.30.20.247",
+    "guides":  "10.30.20.248",
+}
+
+
+def get_home_tunnel_config():
+    """Get current home tunnel ingress configuration."""
+    result = subprocess.run(
+        ["cloudflared", "tunnel", "info", "--output", "json", HOME_TUNNEL_ID],
+        capture_output=True, text=True, timeout=10
+    )
+    if result.returncode == 0 and result.stdout.strip():
+        return json.loads(result.stdout)
+    return None
+
+
+def add_ingress_rules():
+    """Add ingress rules for all 8 publisher sites to the home tunnel."""
+    print("Adding Cloudflare tunnel ingress rules...")
+    
+    for name, ip in SITES.items():
+        hostname = f"{name}.{DOMAIN}"
+        
+        # Check if DNS record already exists
+        print(f"\n  {hostname}:")
+        
+        # Use cloudflared CLI to add route
+        cmd = [
+            "cloudflared", "tunnel", "route", "dns",
+            "--overwrite-dns",
+            HOME_TUNNEL_ID, hostname
+        ]
+        
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        if result.returncode == 0:
+            print(f"    ✅ DNS route created: {hostname} → tunnel")
+        else:
+            error = result.stderr.strip()
+            if "already exists" in error.lower() or "conflict" in error.lower():
+                print(f"    ⚠️  DNS already exists: {hostname}")
+            else:
+                print(f"    ❌ Failed: {error[:200]}")
+                # Fallback: manual DNS via API
+                _add_dns_manual(name, hostname)
+        
+        # Add local ingress mapping using config file approach
+        _add_config_ingress(name, hostname, ip)
+
+
+def _add_dns_manual(name, hostname):
+    """Manual DNS CNAME creation via API."""
+    # This requires a working CF token
+    CF_TOKEN = "cfat_M62Ke6eLXPZnb26qW4FCRL4T4h8l4KrRVLFIRWad9455c3e0"
+    
+    result = subprocess.run([
+        "curl", "-s", "-X", "POST",
+        f"https://api.cloudflare.com/client/v4/zones/{ZONE_ID}/dns_records",
+        "-H", f"Authorization: Bearer {CF_TOKEN}",
+        "-H", "Content-Type: application/json",
+        "-d", json.dumps({
+            "type": "CNAME",
+            "name": name,
+            "content": f"{HOME_TUNNEL_ID}.cfargotunnel.com",
+            "proxied": True,
+            "ttl": 1,
+        }),
+    ], capture_output=True, text=True, timeout=15)
+    
+    try:
+        resp = json.loads(result.stdout)
+        if resp.get("success"):
+            print(f"    ✅ Manual DNS created: {hostname}")
+        else:
+            errors = resp.get("errors", [])
+            for e in errors:
+                if "already exists" in str(e).lower() or e.get("code") == 81053:
+                    print(f"    ⚠️  DNS already exists: {hostname}")
+                    return
+            print(f"    ❌ Manual DNS failed: {errors}")
+    except Exception:
+        print(f"    ❌ API error: {result.stdout[:200]}")
+
+
+def _add_config_ingress(name, hostname, ip):
+    """Add ingress rule to cloudflared config file."""
+    config_path = "/usr/local/etc/cloudflared/config.yml"
+    
+    # Check if cloudflared is running locally
+    result = subprocess.run(
+        ["pgrep", "-f", "cloudflared"], capture_output=True, text=True
+    )
+    
+    if result.returncode != 0:
+        print(f"    ℹ️  cloudflared not running locally — tunnel is on Proxmox CT680")
+        print(f"    ℹ️  Add manually: cloudflared tunnel route dns {HOME_TUNNEL_ID} {hostname}")
+        return
+    
+    # Read existing config
+    try:
+        with open(config_path) as f:
+            config = f.read()
+    except FileNotFoundError:
+        print(f"    ℹ️  No local cloudflared config — tunnel managed elsewhere")
+        return
+    
+    # Check if entry exists
+    if f"hostname: {hostname}" in config:
+        print(f"    ⚠️  Ingress already in config for {hostname}")
+        return
+    
+    # Add ingress rule before the catch-all
+    new_rule = f"""
+  - hostname: {hostname}
+    service: http://{ip}:80"""
+    
+    if "service: http_status:404" in config:
+        config = config.replace(
+            "service: http_status:404",
+            f"{new_rule}\n  - service: http_status:404"
+        )
+    
+    with open(config_path, "w") as f:
+        f.write(config)
+    
+    print(f"    ✅ Ingress added locally for {hostname}")
+    
+    # Restart cloudflared
+    subprocess.run(["brew", "services", "restart", "cloudflared"], 
+                   capture_output=True, timeout=10)
+    print(f"    ✅ cloudflared restarted")
+
+
+def verify_all():
+    """Verify all sites are accessible via Cloudflare tunnel."""
+    print("\n\nVerifying all sites...")
+    time.sleep(5)
+    
+    for name in SITES:
+        url = f"https://{name}.{DOMAIN}"
+        result = subprocess.run(
+            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", url],
+            capture_output=True, text=True, timeout=10
+        )
+        code = result.stdout.strip()
+        status = "✅" if code in ("200", "301", "302") else "❌"
+        print(f"  {status} {url} → HTTP {code}")
+
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("CLOUDFLARE TUNNEL SETUP — Auto Publisher Sites")
+    print("=" * 60)
+    print(f"\nHome Tunnel: {HOME_TUNNEL_ID}")
+    print(f"Domain: {DOMAIN}")
+    print()
+    
+    add_ingress_rules()
+    verify_all()
+    
+    print(f"\n{'='*60}")
+    print("TUNNEL SETUP COMPLETE")
+    print(f"{'='*60}")

core/dashboard.log

@@ -0,0 +1,3 @@
+Admin Dashboard → http://127.0.0.1:5106
+ * Serving Flask app 'app'
+ * Debug mode: off

core/dashboard_error.log

@@ -0,0 +1,4 @@
+WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5106
+Press CTRL+C to quit
+127.0.0.1 - - [03/Aug/2026 21:35:50] "GET /health HTTP/1.1" 200 -

core/deploy_sites.py

@@ -0,0 +1,334 @@
+"""
+Deploy all 8 auto-publisher sites to Proxmox LXC containers.
+Creates CTs, installs nginx, configures cloudflared tunnels, and pushes code.
+"""
+import os
+import sys
+import time
+import json
+import subprocess
+from pathlib import Path
+
+PROXMOX = "root@10.30.20.85"
+PROXMOX_PASS = "czapiewski"
+STORAGE = "poolmaster"
+BRIDGE = "vmbr0"
+GATEWAY = "10.30.20.1"
+TEMPLATE = "local:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst"
+DOMAIN = "thetempleofdoom.com"
+GITEA_HOST = "10.30.20.149:3000"
+GITEA_TOKEN = "dec25a837e0d85573706c4d5d608c2df215f3320"
+
+# Site definitions: name → (ct_id, ip)
+SITES = {
+    "ai":      (135, "10.30.20.240"),
+    "tech":    (136, "10.30.20.241"),
+    "science": (137, "10.30.20.242"),
+    "crypto":  (138, "10.30.20.243"),
+    "linux":   (139, "10.30.20.244"),
+    "gaming":  (140, "10.30.20.246"),
+    "diy":     (141, "10.30.20.247"),
+    "guides":  (142, "10.30.20.248"),
+}
+
+
+def ssh(cmd: str, timeout: int = 30) -> str:
+    """Run command on Proxmox host."""
+    result = subprocess.run(
+        ["ssh", "-o", "StrictHostKeyChecking=accept-new", PROXMOX, cmd],
+        capture_output=True, text=True, timeout=timeout
+    )
+    return result.stdout.strip()
+
+
+def pct_exec(vmid: int, cmd: str, timeout: int = 30) -> str:
+    """Run command inside a CT."""
+    return ssh(f"pct exec {vmid} -- bash -c '{cmd}'", timeout=timeout)
+
+
+def create_ct(name: str, vmid: int, ip: str) -> bool:
+    """Create a Proxmox LXC container."""
+    print(f"  Creating CT {vmid} ({name}) at {ip}...")
+    
+    # Check if CT already exists
+    existing = ssh(f"pct list | grep '^{vmid} '")
+    if existing:
+        print(f"    CT {vmid} already exists — skipping creation")
+        return True
+    
+    cmd = (
+        f"pct create {vmid} {TEMPLATE} "
+        f"--hostname {name}-publisher "
+        f"--storage {STORAGE} "
+        f"--memory 2048 --swap 512 --cores 2 "
+        f"--net0 name=eth0,bridge={BRIDGE},ip={ip}/24,gw={GATEWAY} "
+        f"--unprivileged 1 --password {PROXMOX_PASS} "
+        f"--features nesting=1 --onboot 1"
+    )
+    out = ssh(cmd)
+    print(f"    Created: {out}")
+    
+    # Start it
+    ssh(f"pct start {vmid}")
+    time.sleep(8)
+    
+    # Get assigned IP
+    ip_check = ssh(f"pct exec {vmid} -- hostname -I")
+    print(f"    IP: {ip_check}")
+    
+    return True
+
+
+def setup_ct(name: str, vmid: int, ip: str) -> bool:
+    """Install nginx and configure the site on a CT."""
+    print(f"  Setting up CT {vmid} ({name})...")
+    
+    # Update packages
+    pct_exec(vmid, "apt update -qq && apt install -y -qq nginx curl python3 python3-pip git 2>&1 | tail -5",
+             timeout=120)
+    
+    # Create web root
+    pct_exec(vmid, "mkdir -p /var/www/html/articles /var/www/html/assets/images")
+    
+    # Copy shared assets
+    local_assets = Path(__file__).resolve().parent.parent / "shared" / "assets"
+    if local_assets.exists():
+        # Create tarball of shared assets
+        subprocess.run(
+            f"cd {local_assets.parent} && tar czf /tmp/shared_assets.tar.gz assets/",
+            shell=True, capture_output=True
+        )
+        subprocess.run(
+            f"scp /tmp/shared_assets.tar.gz {PROXMOX}:/tmp/shared_assets.tar.gz",
+            shell=True, capture_output=True
+        )
+        ssh(f"pct push {vmid} /tmp/shared_assets.tar.gz /tmp/shared_assets.tar.gz")
+        pct_exec(vmid, "cd /var/www/html && tar xzf /tmp/shared_assets.tar.gz")
+    
+    # Create nginx config
+    nginx_conf = f"""server {{
+    listen 80 default_server;
+    server_name {name}.{DOMAIN};
+    root /var/www/html;
+    index index.html;
+
+    location / {{
+        try_files $uri $uri/ $uri.html =404;
+    }}
+
+    location /assets/ {{
+        expires 30d;
+        add_header Cache-Control "public, immutable";
+    }}
+
+    location /rss.xml {{
+        add_header Content-Type "application/rss+xml";
+    }}
+
+    location /sitemap.xml {{
+        add_header Content-Type "application/xml";
+    }}
+
+    # Gzip
+    gzip on;
+    gzip_types text/plain text/html text/css application/json application/javascript text/xml application/xml;
+    gzip_min_length 1000;
+}}"""
+    
+    # Write nginx config via base64 to avoid quoting issues
+    import base64
+    encoded = base64.b64encode(nginx_conf.encode()).decode()
+    pct_exec(vmid, f"echo '{encoded}' | base64 -d > /etc/nginx/sites-available/default")
+    pct_exec(vmid, "rm -f /etc/nginx/sites-enabled/default && "
+                   "ln -s /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default")
+    
+    # Verify and restart nginx
+    result = pct_exec(vmid, "nginx -t 2>&1 && systemctl restart nginx && echo 'NGINX_OK'")
+    print(f"    Nginx: {'OK' if 'NGINX_OK' in result else 'FAILED'}")
+    
+    # Create a simple health endpoint
+    pct_exec(vmid, f"echo '<html><body><h1>{name}.{DOMAIN}</h1><p>Auto Publisher — coming soon</p></body></html>' > /var/www/html/index.html")
+    
+    return True
+
+
+def create_gitea_repo(name: str) -> bool:
+    """Create a Gitea repository for a site."""
+    print(f"  Creating Gitea repo: {name}-publisher...")
+    
+    # Check if repo already exists
+    result = subprocess.run(
+        ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
+         f"http://{GITEA_HOST}/api/v1/repos/drjones/{name}-publisher"],
+        capture_output=True, text=True
+    )
+    if result.stdout.strip() == "200":
+        print(f"    Repo {name}-publisher already exists")
+        return True
+    
+    # Create via API
+    result = subprocess.run(
+        ["curl", "-s", "-X", "POST",
+         f"http://{GITEA_HOST}/api/v1/user/repos",
+         "-H", "Content-Type: application/json",
+         "-H", f"Authorization: token {GITEA_TOKEN}",
+         "-d", json.dumps({
+             "name": f"{name}-publisher",
+             "description": f"Auto Publisher site: {name}.thetempleofdoom.com",
+             "private": False,
+             "auto_init": True,
+         })],
+        capture_output=True, text=True
+    )
+    
+    if "200" in result.stdout or "201" in result.stdout or "409" in result.stdout:
+        print(f"    Repo {name}-publisher created")
+        return True
+    else:
+        print(f"    Gitea API failed: {result.stdout[:200]}")
+        # Fallback: create via filesystem + SQLite
+        ssh(f"pct exec 525 -- mkdir -p /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git")
+        ssh(f"pct exec 525 -- git init --bare /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git")
+        ssh(f"pct exec 525 -- chown -R gitea:gitea /var/lib/gitea/data/gitea-repositories/drjones/{name}-publisher.git")
+        return True
+
+
+def verify_site(name: str, ip: str) -> bool:
+    """Verify a site is serving content."""
+    try:
+        result = subprocess.run(
+            ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
+             f"http://{ip}:80/"],
+            capture_output=True, text=True, timeout=5
+        )
+        code = result.stdout.strip()
+        print(f"    HTTP status: {code}")
+        return code == "200"
+    except Exception as e:
+        print(f"    Verify failed: {e}")
+        return False
+
+
+def deploy_site_code(name: str, vmid: int) -> bool:
+    """Push site code to CT and to Gitea."""
+    print(f"  Pushing code to CT {vmid}...")
+    
+    site_dir = Path(__file__).resolve().parent.parent / "sites" / name
+    
+    # Build a placeholder site if directory is empty
+    if not site_dir.exists() or not list(site_dir.glob("*.html")):
+        # Create placeholder
+        site_dir.mkdir(parents=True, exist_ok=True)
+        (site_dir / "articles").mkdir(exist_ok=True)
+        
+        placeholder = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{name}.thetempleofdoom.com</title>
+    <link rel="stylesheet" href="/assets/style.css">
+</head>
+<body>
+    <header>
+        <nav>
+            <a href="/" class="logo">{name}.thetempleofdoom.com</a>
+        </nav>
+    </header>
+    <main>
+        <section class="hero">
+            <h1>{name.title()} Insights & Guides</h1>
+            <p>Expert {name} articles, tutorials, and deep dives. Updated daily.</p>
+            <p style="margin-top:2rem;color:var(--text2);">🚀 First article coming soon...</p>
+        </section>
+        <section class="articles-grid">
+            <div class="card">
+                <h2>Coming Soon</h2>
+                <p class="meta">The autonomous publishing pipeline is being set up.</p>
+                <p>Fresh {name} content will be published here automatically every day.</p>
+            </div>
+        </section>
+    </main>
+    <footer class="site-footer">
+        <p>&copy; 2026 {name}.thetempleofdoom.com</p>
+    </footer>
+</body>
+</html>"""
+        (site_dir / "index.html").write_text(placeholder)
+    
+    # Tar and push
+    subprocess.run(
+        f"cd {site_dir.parent} && tar czf /tmp/{name}_site.tar.gz {name}/",
+        shell=True, capture_output=True
+    )
+    subprocess.run(
+        f"scp /tmp/{name}_site.tar.gz {PROXMOX}:/tmp/{name}_site.tar.gz",
+        shell=True, capture_output=True
+    )
+    ssh(f"pct push {vmid} /tmp/{name}_site.tar.gz /tmp/{name}_site.tar.gz")
+    pct_exec(vmid, f"cd /var/www/html && tar xzf /tmp/{name}_site.tar.gz --strip-components=1 && chown -R www-data:www-data /var/www/html")
+    
+    # Push to Gitea
+    git_dir = site_dir
+    subprocess.run(
+        f"cd {git_dir} && "
+        f"git init 2>/dev/null; "
+        f"git config user.email 'drjones@thetempleofdoom.com'; "
+        f"git config user.name 'drjones'; "
+        f"git add . 2>/dev/null; "
+        f"git commit -m 'Initial: {name} publisher site' 2>/dev/null; "
+        f"git remote remove origin 2>/dev/null; "
+        f"git remote add origin http://drjones:{GITEA_TOKEN}@{GITEA_HOST}/drjones/{name}-publisher.git; "
+        f"git push -u origin main --force 2>&1",
+        shell=True, capture_output=True
+    )
+    
+    return True
+
+
+def main():
+    print("=" * 60)
+    print("AUTO PUBLISHER — Site Deployment")
+    print("=" * 60)
+    print()
+    
+    for name, (vmid, ip) in SITES.items():
+        print(f"\n{'='*40}")
+        print(f"  📡 {name}.thetempleofdoom.com (CT {vmid}, {ip})")
+        print(f"{'='*40}")
+        
+        # Step 1: Create CT
+        if not create_ct(name, vmid, ip):
+            print(f"  ❌ Failed to create CT for {name}")
+            continue
+        
+        # Step 2: Setup nginx
+        if not setup_ct(name, vmid, ip):
+            print(f"  ❌ Failed to setup {name}")
+            continue
+        
+        # Step 3: Create Gitea repo
+        create_gitea_repo(name)
+        
+        # Step 4: Push initial code
+        deploy_site_code(name, vmid)
+        
+        # Step 5: Verify
+        time.sleep(2)
+        if verify_site(name, ip):
+            print(f"  ✅ {name}.thetempleofdoom.com is LIVE (http://{ip}:80)")
+        else:
+            print(f"  ⚠️  {name} may need nginx restart")
+    
+    print(f"\n{'='*60}")
+    print("DEPLOYMENT COMPLETE")
+    print(f"{'='*60}")
+    print("\nSites deployed:")
+    for name, (vmid, ip) in SITES.items():
+        print(f"  {name}.thetempleofdoom.com → CT {vmid} @ {ip}")
+    print("\nNext: Set up Cloudflare tunnels for public access")
+    print("Run: python3 core/cloudflare_setup.py")
+
+
+if __name__ == "__main__":
+    main()

core/orchestrator.log

@@ -0,0 +1,5 @@
+2026-08-03 21:38:15,046 [INFO] orchestrator: Starting trend discovery...
+2026-08-03 21:38:19,444 [WARNING] orchestrator: Seasonal topic generation failed: ollama_json() got an unexpected keyword argument 'temperature'
+2026-08-03 21:38:19,445 [WARNING] orchestrator: Topic scoring failed: ollama_json() got an unexpected keyword argument 'temperature'
+2026-08-03 21:38:19,449 [INFO] orchestrator: Discovered 0 topics, stored top 25
+2026-08-03 21:38:30,350 [INFO] orchestrator: Starting trend discovery...