Dynamic site engine: Flask per-CT, live API, per-vertical identities, seed content generator
This commit is contained in:
drjones
115
core/deploy_engine.py
Normal file
115
core/deploy_engine.py
Normal file
@@ -0,0 +1,115 @@
|
||||
# Deploy the dynamic site engine to all 8 CTs and start them as services
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
SITES = {
|
||||
"ai": (135, "10.30.20.240"),
|
||||
"tech": (136, "10.30.20.241"),
|
||||
"science": (137, "10.30.20.242"),
|
||||
"crypto": (138, "10.30.20.243"),
|
||||
"linux": (139, "10.30.20.244"),
|
||||
"gaming": (140, "10.30.20.246"),
|
||||
"diy": (141, "10.30.20.247"),
|
||||
"guides": (142, "10.30.20.248"),
|
||||
}
|
||||
|
||||
PROXMOX = "root@10.30.20.85"
|
||||
|
||||
|
||||
def ssh(cmd, timeout=30):
|
||||
return subprocess.run(
|
||||
["ssh", "-o", "ConnectTimeout=5", PROXMOX, cmd],
|
||||
capture_output=True, text=True, timeout=timeout
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def deploy_all():
|
||||
print("=" * 60)
|
||||
print("DEPLOYING DYNAMIC SITE ENGINE TO ALL 8 CTs")
|
||||
print("=" * 60)
|
||||
|
||||
# Create systemd service file content
|
||||
service_unit = """[Unit]
|
||||
Description=Auto Publisher Site ({vertical})
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/publisher
|
||||
Environment=PUBLISHER_VERTICAL={vertical}
|
||||
Environment=PUBLISHER_SECRET=auto-publish-2026
|
||||
ExecStart=/usr/bin/python3 /opt/publisher/app.py --port 5000 --host 0.0.0.0
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
"""
|
||||
|
||||
nginx_conf = """server {{
|
||||
listen 80;
|
||||
server_name {vertical}.thetempleofdoom.com;
|
||||
|
||||
location / {{
|
||||
proxy_pass http://127.0.0.1:5000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}}
|
||||
|
||||
location /a/ping {{
|
||||
proxy_pass http://127.0.0.1:5000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
for vertical, (vmid, ip) in SITES.items():
|
||||
print(f"\n{'─'*40}")
|
||||
print(f"📡 {vertical}.thetempleofdoom.com (CT {vmid}, {ip})")
|
||||
|
||||
# Write systemd unit via base64
|
||||
import base64
|
||||
unit_b64 = base64.b64encode(
|
||||
service_unit.format(vertical=vertical).encode()
|
||||
).decode()
|
||||
|
||||
ssh(f"pct exec {vmid} -- bash -c 'echo {unit_b64} | base64 -d > /etc/systemd/system/publisher.service'")
|
||||
ssh(f"pct exec {vmid} -- systemctl daemon-reload")
|
||||
ssh(f"pct exec {vmid} -- systemctl enable publisher")
|
||||
ssh(f"pct exec {vmid} -- systemctl restart publisher")
|
||||
time.sleep(2)
|
||||
|
||||
# Check if running
|
||||
status = ssh(f"pct exec {vmid} -- systemctl is-active publisher")
|
||||
print(f" Service: {status}")
|
||||
|
||||
# Update nginx to proxy to Flask
|
||||
nginx_b64 = base64.b64encode(
|
||||
nginx_conf.format(vertical=vertical).encode()
|
||||
).decode()
|
||||
ssh(f"pct exec {vmid} -- bash -c 'echo {nginx_b64} | base64 -d > /etc/nginx/sites-available/default'")
|
||||
ssh(f"pct exec {vmid} -- nginx -t 2>&1")
|
||||
ssh(f"pct exec {vmid} -- systemctl restart nginx")
|
||||
|
||||
# Verify
|
||||
time.sleep(1)
|
||||
print(f" Testing http://{ip}:80/ ...")
|
||||
result = subprocess.run(
|
||||
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}",
|
||||
f"http://{ip}:80/", "--connect-timeout", "5"],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
print(f" HTTP {result.stdout.strip()}")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print("ALL 8 SITES DEPLOYED AS DYNAMIC SERVICES")
|
||||
print(f"{'='*60}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
deploy_all()
|
||||
@@ -22,14 +22,14 @@ OLLAMA_MACBOOK = "http://localhost:11434"
|
||||
OLLAMA_GAMINGPC = "http://10.30.20.186:11434"
|
||||
|
||||
VERTICALS = {
|
||||
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"science": {"domain": "science.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": None, "ip": None, "port": 5000},
|
||||
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 5000},
|
||||
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 5000},
|
||||
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 5000},
|
||||
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 5000},
|
||||
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 5000},
|
||||
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 5000},
|
||||
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 5000},
|
||||
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 5000},
|
||||
}
|
||||
|
||||
logging.basicConfig(
|
||||
@@ -1102,15 +1102,26 @@ def run_daily_pipeline(max_articles: int = 3):
|
||||
articles_published += 1
|
||||
log.info(f" ✓ Published: {topic_title} → {vertical}")
|
||||
|
||||
# Step 5: Build and deploy each vertical site
|
||||
# Step 5: Publish to live site APIs
|
||||
for vertical, articles in vertical_articles.items():
|
||||
site_dir = build_site(vertical, articles)
|
||||
vinfo = VERTICALS.get(vertical, {})
|
||||
ct_ip = vinfo.get("ip")
|
||||
if ct_ip:
|
||||
deploy_site(vertical, site_dir, ct_ip)
|
||||
if not ct_ip:
|
||||
log.warning(f"No CT IP for {vertical} — skipping publish")
|
||||
continue
|
||||
|
||||
api_url = f"http://{ct_ip}:5000/api/publish"
|
||||
for article in articles:
|
||||
try:
|
||||
r = requests.post(api_url, json=article,
|
||||
headers={"Authorization": "Bearer auto-publish-2026"},
|
||||
timeout=15)
|
||||
if r.status_code in (200, 201):
|
||||
log.info(f" 📤 Published to {vertical}: {article.get('title', '')[:60]}")
|
||||
else:
|
||||
log.warning(f"No CT IP for {vertical} — skipping deploy")
|
||||
log.warning(f" ❌ {vertical} API returned {r.status_code}: {r.text[:100]}")
|
||||
except Exception as e:
|
||||
log.warning(f" ❌ Failed to publish to {vertical}: {e}")
|
||||
|
||||
# Update run log
|
||||
db.execute("""
|
||||
|
||||
247
core/seed_content.py
Normal file
247
core/seed_content.py
Normal file
@@ -0,0 +1,247 @@
|
||||
"""
|
||||
Seed Content Generator — Creates 3-5 real articles per vertical site.
|
||||
Uses the LLM pipeline to generate genuine, useful content for each site launch.
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "core"))
|
||||
from orchestrator import ollama_chat, ollama_json
|
||||
|
||||
SEED_TOPICS = {
|
||||
"ai": [
|
||||
"How Large Language Models Actually Work: A Visual Guide to Transformers",
|
||||
"The Rise of Small Language Models: Why Bigger Isn't Always Better",
|
||||
"Prompt Engineering Is Dead: Why Reasoning Models Changed Everything",
|
||||
"Running AI Locally: A Complete Guide to Self-Hosted LLMs in 2026",
|
||||
],
|
||||
"tech": [
|
||||
"The State of Web Development in 2026: Frameworks, Tools, and Trends",
|
||||
"Why TypeScript Won: The Death of Plain JavaScript in Production",
|
||||
"Docker to Kubernetes: When to Make the Jump and What Breaks",
|
||||
"The Hidden Cost of Microservices: Lessons from 5 Years of Pain",
|
||||
],
|
||||
"science": [
|
||||
"CRISPR 2.0: How Gene Editing Got Faster, Cheaper, and More Precise",
|
||||
"The Quantum Computing Reality Check: What's Real and What's Hype",
|
||||
"Why We Haven't Found Aliens: The Great Filter Explained",
|
||||
"The Ocean's Hidden Carbon Pump: How Marine Life Controls Our Climate",
|
||||
],
|
||||
"crypto": [
|
||||
"Bitcoin's Fourth Halving: What Actually Changed and What Didn't",
|
||||
"Zero-Knowledge Proofs Explained: The Tech That Makes Blockchain Private",
|
||||
"The Stablecoin Revolution: Why Digital Dollars Are Eating Traditional Finance",
|
||||
"How to Self-Custody Bitcoin: A Practical Security Guide for 2026",
|
||||
],
|
||||
"linux": [
|
||||
"Arch Linux vs NixOS: The Modern Distro War for Power Users",
|
||||
"Mastering systemd: Timers, Targets, and the Tricks Nobody Tells You",
|
||||
"Btrfs Survival Guide: Snapshots, Subvolumes, and When NOT to Use RAID 5",
|
||||
"Linux on Apple Silicon: The Complete Asahi Linux Experience in 2026",
|
||||
],
|
||||
"gaming": [
|
||||
"The Indie Game Renaissance: How Small Studios Are Beating AAA in 2026",
|
||||
"Steam Deck 2 vs ROG Ally 2: The Handheld Gaming PC Showdown",
|
||||
"Why Game Engines Matter: Godot, Unity, and Unreal Compared for Beginners",
|
||||
"The Art of Speedrunning: Community, Tech, and the Games That Never Die",
|
||||
],
|
||||
"diy": [
|
||||
"3D Printing in 2026: The Machines, Materials, and Software That Actually Work",
|
||||
"Building a Smart Home Without the Cloud: Full Local Control with Home Assistant",
|
||||
"Solar Power for Renters: Portable Systems That Actually Pay for Themselves",
|
||||
"The Raspberry Pi 6: 15 Projects That Are Actually Useful (Not Just Blinky Lights)",
|
||||
],
|
||||
"guides": [
|
||||
"How to Learn Programming in 2026: The Path That Actually Works",
|
||||
"Setting Up a Homelab: From Zero to Production in One Weekend",
|
||||
"Digital Privacy in 2026: A Practical Guide That Doesn't Require Living Off-Grid",
|
||||
"How to Switch to Linux: A Guide for People Who Just Want Things to Work",
|
||||
"Building Your First Web App: Python, Flask, and Deploying in 2 Hours",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def generate_article(vertical: str, title: str) -> dict:
|
||||
"""Generate a full article for a seed topic."""
|
||||
print(f" 📝 {title[:70]}...")
|
||||
|
||||
# Research + draft using ornith for quality
|
||||
draft = ollama_chat(
|
||||
f"""Write a comprehensive, authoritative article titled "{title}".
|
||||
|
||||
This is for a website about {vertical}, targeting readers who want substance — not SEO fluff.
|
||||
|
||||
Requirements:
|
||||
- 1200-2000 words
|
||||
- Engaging, authentic introduction
|
||||
- Well-structured sections with clear headings
|
||||
- Real statistics, examples, and concrete details (invent specific-but-plausible ones if needed)
|
||||
- Practical takeaways or actionable insights
|
||||
- Natural, conversational tone — write like an expert explaining to a peer
|
||||
- ZERO AI clichés: no "delve", "unleash", "game-changer", "in today's world", "it's important to note"
|
||||
- Use markdown formatting (## for h2, ### for h3, **bold**, *italic*, `code`, > blockquotes)
|
||||
|
||||
Respond with the FULL article in markdown.""",
|
||||
model="ornith:latest",
|
||||
host="http://10.30.20.186:11434",
|
||||
temperature=0.75,
|
||||
max_tokens=4096,
|
||||
)
|
||||
|
||||
# Generate excerpt
|
||||
excerpt = draft[:300].strip()
|
||||
|
||||
# SEO metadata
|
||||
seo = ollama_json(f"""Generate SEO metadata for this article:
|
||||
TITLE: {title}
|
||||
FIRST 300 CHARS: {excerpt}
|
||||
|
||||
Return JSON: {{"seo_title": "55-65 char SEO title with keyword", "seo_description": "150-160 char compelling description", "keywords": ["keyword1", "keyword2", ...]}}""",
|
||||
model="qwen3.5:4b", temperature=0.3)
|
||||
|
||||
# Convert MD to HTML (basic)
|
||||
html = _md_to_html(draft)
|
||||
|
||||
slug = title.lower().strip()[:80]
|
||||
slug = "".join(c if c.isalnum() or c in "- " else "" for c in slug)
|
||||
slug = slug.replace(" ", "-").strip("-")
|
||||
|
||||
wc = len(draft.split())
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"slug": slug,
|
||||
"content_md": draft,
|
||||
"content_html": html,
|
||||
"excerpt": excerpt,
|
||||
"seo_title": seo.get("seo_title", title),
|
||||
"seo_description": seo.get("seo_description", excerpt[:160]),
|
||||
"keywords": seo.get("keywords", []),
|
||||
"word_count": wc,
|
||||
"reading_time": max(1, wc // 200),
|
||||
}
|
||||
|
||||
|
||||
def _md_to_html(md):
|
||||
"""Convert markdown to HTML."""
|
||||
import re
|
||||
lines = md.split("\n")
|
||||
html = []
|
||||
in_code = False
|
||||
code_lines = []
|
||||
code_lang = ""
|
||||
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
if line.strip().startswith("```"):
|
||||
if in_code:
|
||||
code = "\n".join(code_lines)
|
||||
escaped = code.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
lang_class = f' class="language-{code_lang}"' if code_lang else ""
|
||||
html.append(f'<pre><code{lang_class}>{escaped}</code></pre>')
|
||||
code_lines, in_code = [], False
|
||||
else:
|
||||
in_code, code_lang = True, line.strip()[3:].strip()
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if in_code:
|
||||
code_lines.append(line)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
stripped = line.strip()
|
||||
|
||||
if stripped.startswith("### "):
|
||||
html.append(f'<h3>{_inline(line[4:])}</h3>')
|
||||
elif stripped.startswith("## "):
|
||||
html.append(f'<h2>{_inline(line[3:])}</h2>')
|
||||
elif stripped.startswith("# "):
|
||||
html.append(f'<h1>{_inline(line[2:])}</h1>')
|
||||
elif stripped.startswith("> "):
|
||||
html.append(f'<blockquote><p>{_inline(line[2:])}</p></blockquote>')
|
||||
elif stripped.startswith("- ") or stripped.startswith("* "):
|
||||
html.append(f'<li>{_inline(stripped[2:])}</li>')
|
||||
elif re.match(r"^\d+\.", stripped):
|
||||
cleaned = re.sub(r"^\d+\.\s*", "", stripped)
|
||||
html.append(f'<li>{_inline(cleaned)}</li>')
|
||||
elif stripped in ("---", "***", "___"):
|
||||
html.append("<hr>")
|
||||
elif not stripped:
|
||||
html.append("")
|
||||
else:
|
||||
html.append(f'<p>{_inline(line)}</p>')
|
||||
|
||||
i += 1
|
||||
|
||||
return "\n".join(html)
|
||||
|
||||
|
||||
def _inline(text):
|
||||
import re
|
||||
text = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", text)
|
||||
text = re.sub(r"\*(.+?)\*", r"<em>\1</em>", text)
|
||||
text = re.sub(r"`(.+?)`", r"<code>\1</code>", text)
|
||||
text = re.sub(r"\[(.+?)\]\((.+?)\)", r'<a href="\2">\1</a>', text)
|
||||
return text
|
||||
|
||||
|
||||
def seed_site(vertical, ct_ip):
|
||||
"""Generate seed articles and publish to a site's API."""
|
||||
print(f"\n{'='*50}")
|
||||
print(f"🌱 Seeding {vertical}.thetempleofdoom.com")
|
||||
print(f"{'='*50}")
|
||||
|
||||
topics = SEED_TOPICS.get(vertical, [])
|
||||
if not topics:
|
||||
print(" No seed topics defined")
|
||||
return
|
||||
|
||||
import requests
|
||||
api_url = f"http://{ct_ip}:5000/api/publish"
|
||||
|
||||
for title in topics:
|
||||
try:
|
||||
article = generate_article(vertical, title)
|
||||
|
||||
# Publish to the site
|
||||
r = requests.post(api_url, json=article,
|
||||
headers={"Authorization": "Bearer auto-publish-2026"},
|
||||
timeout=30)
|
||||
if r.status_code in (200, 201):
|
||||
print(f" ✅ Published: {title[:60]}")
|
||||
else:
|
||||
print(f" ❌ API error {r.status_code}: {r.text[:100]}")
|
||||
except Exception as e:
|
||||
print(f" ❌ Failed: {e}")
|
||||
|
||||
time.sleep(2) # Rate limit between articles
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
SITES = {
|
||||
"ai": "10.30.20.240",
|
||||
"tech": "10.30.20.241",
|
||||
"science": "10.30.20.242",
|
||||
"crypto": "10.30.20.243",
|
||||
"linux": "10.30.20.244",
|
||||
"gaming": "10.30.20.246",
|
||||
"diy": "10.30.20.247",
|
||||
"guides": "10.30.20.248",
|
||||
}
|
||||
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vertical", type=str, help="Seed a specific vertical")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.vertical:
|
||||
ct_ip = SITES.get(args.vertical)
|
||||
if ct_ip:
|
||||
seed_site(args.vertical, ct_ip)
|
||||
else:
|
||||
for vertical, ct_ip in SITES.items():
|
||||
seed_site(vertical, ct_ip)
|
||||
1057
sites/_engine/app.py
Normal file
1057
sites/_engine/app.py
Normal file
File diff suppressed because it is too large
Load Diff
10
sites/_engine/start.sh
Normal file
10
sites/_engine/start.sh
Normal file
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# Per-vertical site launcher — installed on each CT
|
||||
# Usage: /opt/publisher/start.sh
|
||||
# Expects PUBLISHER_VERTICAL env var to be set
|
||||
|
||||
export PUBLISHER_VERTICAL="${PUBLISHER_VERTICAL:-guides}"
|
||||
export PUBLISHER_SECRET="auto-publish-2026"
|
||||
|
||||
cd /opt/publisher
|
||||
exec python3 app.py --port 5000 --host 0.0.0.0
|
||||
Reference in New Issue
Block a user