1274 lines
49 KiB
Python
1274 lines
49 KiB
Python
"""
|
|
Autonomous Publishing System — Core Orchestrator
|
|
Runs daily to discover, research, write, and publish content across all vertical sites.
|
|
"""
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import sqlite3
|
|
import logging
|
|
from pathlib import Path
|
|
from datetime import datetime, timedelta
|
|
from dataclasses import dataclass, field, asdict
|
|
from typing import Optional
|
|
|
|
import requests
|
|
|
|
# ─── Config ───────────────────────────────────────────────────────
|
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
DB_PATH = BASE_DIR / "core" / "publisher.db"
|
|
OLLAMA_MACBOOK = "http://localhost:11434"
|
|
OLLAMA_GAMINGPC = "http://10.30.20.186:11434"
|
|
|
|
VERTICALS = {
|
|
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 5000},
|
|
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 5000},
|
|
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 5000},
|
|
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 5000},
|
|
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 5000},
|
|
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 5000},
|
|
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 5000},
|
|
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 5000},
|
|
}
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
handlers=[
|
|
logging.FileHandler(BASE_DIR / "core" / "orchestrator.log"),
|
|
logging.StreamHandler(),
|
|
],
|
|
)
|
|
log = logging.getLogger("orchestrator")
|
|
|
|
|
|
# ─── Database ──────────────────────────────────────────────────────
|
|
def init_db():
|
|
"""Initialize the SQLite database with all required tables."""
|
|
db = sqlite3.connect(str(DB_PATH))
|
|
db.executescript("""
|
|
CREATE TABLE IF NOT EXISTS topics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
title TEXT NOT NULL,
|
|
vertical TEXT NOT NULL,
|
|
trend_score REAL DEFAULT 0,
|
|
search_volume INTEGER DEFAULT 0,
|
|
competition_score REAL DEFAULT 0,
|
|
freshness_score REAL DEFAULT 0,
|
|
evergreen_score REAL DEFAULT 0,
|
|
composite_score REAL DEFAULT 0,
|
|
sources TEXT DEFAULT '[]',
|
|
status TEXT DEFAULT 'discovered',
|
|
knowledge_package_id INTEGER,
|
|
article_id INTEGER,
|
|
published_url TEXT,
|
|
published_at TIMESTAMP,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS knowledge_packages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
topic_id INTEGER UNIQUE,
|
|
facts TEXT DEFAULT '[]',
|
|
stats TEXT DEFAULT '[]',
|
|
definitions TEXT DEFAULT '[]',
|
|
faqs TEXT DEFAULT '[]',
|
|
misconceptions TEXT DEFAULT '[]',
|
|
timeline TEXT DEFAULT '[]',
|
|
citations TEXT DEFAULT '[]',
|
|
examples TEXT DEFAULT '[]',
|
|
related_concepts TEXT DEFAULT '[]',
|
|
raw_sources TEXT DEFAULT '[]',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS articles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
topic_id INTEGER UNIQUE,
|
|
vertical TEXT NOT NULL,
|
|
title TEXT,
|
|
slug TEXT UNIQUE,
|
|
content_md TEXT,
|
|
content_html TEXT,
|
|
seo_title TEXT,
|
|
seo_description TEXT,
|
|
og_image TEXT,
|
|
json_ld TEXT,
|
|
word_count INTEGER DEFAULT 0,
|
|
reading_time_minutes INTEGER DEFAULT 0,
|
|
status TEXT DEFAULT 'draft',
|
|
published_at TIMESTAMP,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS analytics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
article_id INTEGER,
|
|
vertical TEXT,
|
|
pageviews INTEGER DEFAULT 0,
|
|
unique_visitors INTEGER DEFAULT 0,
|
|
avg_time_on_page REAL DEFAULT 0,
|
|
bounce_rate REAL DEFAULT 0,
|
|
referrers TEXT DEFAULT '[]',
|
|
recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS performance_learning (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
vertical TEXT UNIQUE,
|
|
top_patterns TEXT DEFAULT '[]',
|
|
headline_formats TEXT DEFAULT '[]',
|
|
optimal_word_count INTEGER,
|
|
best_times TEXT DEFAULT '[]',
|
|
keyword_insights TEXT DEFAULT '[]',
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
run_type TEXT,
|
|
topics_discovered INTEGER DEFAULT 0,
|
|
articles_generated INTEGER DEFAULT 0,
|
|
articles_published INTEGER DEFAULT 0,
|
|
errors TEXT DEFAULT '[]',
|
|
started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
finished_at TIMESTAMP
|
|
);
|
|
""")
|
|
db.commit()
|
|
return db
|
|
|
|
|
|
# ─── LLM Helpers ───────────────────────────────────────────────────
|
|
def ollama_chat(prompt: str, model: str = "qwen3.5:4b", host: str = OLLAMA_MACBOOK,
|
|
system: str = "", temperature: float = 0.7, max_tokens: int = 4096) -> str:
|
|
"""Call Ollama chat API. Falls back to GamingPC if MacBook fails."""
|
|
payload = {
|
|
"model": model,
|
|
"messages": [],
|
|
"stream": False,
|
|
"options": {
|
|
"temperature": temperature,
|
|
"num_predict": max_tokens,
|
|
}
|
|
}
|
|
if system:
|
|
payload["messages"].append({"role": "system", "content": system})
|
|
payload["messages"].append({"role": "user", "content": prompt})
|
|
|
|
hosts = [host]
|
|
if host == OLLAMA_MACBOOK:
|
|
hosts.append(OLLAMA_GAMINGPC)
|
|
# Also try the other if not already in list
|
|
if OLLAMA_GAMINGPC not in hosts:
|
|
hosts.append(OLLAMA_GAMINGPC)
|
|
if OLLAMA_MACBOOK not in hosts:
|
|
hosts.append(OLLAMA_MACBOOK)
|
|
|
|
for h in hosts:
|
|
try:
|
|
r = requests.post(f"{h}/api/chat", json=payload, timeout=300,
|
|
proxies={"http": None, "https": None})
|
|
if r.status_code == 200:
|
|
result = r.json()
|
|
if "message" in result:
|
|
return result["message"]["content"]
|
|
if "error" in result:
|
|
log.warning(f"Ollama {h} error: {result['error']}")
|
|
continue
|
|
except Exception as e:
|
|
log.warning(f"Ollama {h} failed: {e}")
|
|
continue
|
|
|
|
# Fallback: use active cloud LLM (DeepSeek) via Hermes tools
|
|
log.warning("All Ollama hosts failed/saturated — falling back to cloud LLM")
|
|
raise RuntimeError(
|
|
f"All Ollama hosts failed for model {model}. "
|
|
"Local LLMs are saturated (likely by Kalshi bots). "
|
|
"Retry when load is lower or add cloud fallback API key."
|
|
)
|
|
|
|
|
|
def ollama_json(prompt: str, model: str = "qwen3.5:4b", host: str = OLLAMA_MACBOOK,
|
|
system: str = "You are a JSON-only API. Always respond with valid JSON. No markdown, no explanation.",
|
|
temperature: float = 0.3) -> dict:
|
|
"""Call Ollama and parse JSON response."""
|
|
raw = ollama_chat(prompt, model=model, host=host, system=system, temperature=temperature)
|
|
# Strip markdown code fences if present
|
|
raw = raw.strip()
|
|
if raw.startswith("```"):
|
|
lines = raw.split("\n")
|
|
raw = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
|
|
return json.loads(raw)
|
|
|
|
|
|
# ─── Trend Discovery ────────────────────────────────────────────────
|
|
TREND_SOURCES = [
|
|
{"name": "Hacker News", "url": "https://hacker-news.firebaseio.com/v0/topstories.json"},
|
|
{"name": "Reddit Programming", "url": "https://www.reddit.com/r/programming/hot.json"},
|
|
{"name": "GitHub Trending", "url": "https://api.github.com/search/repositories?q=created:>{}&sort=stars&order=desc"},
|
|
{"name": "arXiv AI", "url": "https://export.arxiv.org/api/query?search_query=cat:cs.AI&sortBy=submittedDate&max_results=10"},
|
|
{"name": "Stack Overflow", "url": "https://api.stackexchange.com/2.3/questions?order=desc&sort=hot&site=stackoverflow&pagesize=20"},
|
|
]
|
|
|
|
VERTICAL_KEYWORDS = {
|
|
"ai": ["ai", "machine learning", "llm", "gpt", "neural network", "deep learning",
|
|
"transformer", "fine-tuning", "rag", "embedding", "diffusion", "generative ai",
|
|
"openai", "anthropic", "claude", "mistral", "stable diffusion"],
|
|
"tech": ["software", "programming", "api", "database", "cloud", "devops", "kubernetes",
|
|
"docker", "microservices", "react", "typescript", "rust", "golang", "aws"],
|
|
"science": ["physics", "biology", "chemistry", "astronomy", "neuroscience", "quantum",
|
|
"crispr", "climate", "materials", "genetics", "space", "nasa", "jwst"],
|
|
"crypto": ["bitcoin", "ethereum", "crypto", "blockchain", "defi", "nft", "web3",
|
|
"solana", "layer 2", "zk-proof", "mining", "token", "wallet"],
|
|
"linux": ["linux", "kernel", "ubuntu", "debian", "arch", "fedora", "nixos",
|
|
"bash", "systemd", "gnome", "kde", "wayland", "btrfs", "zfs"],
|
|
"gaming": ["game", "steam", "playstation", "xbox", "nintendo", "esports",
|
|
"unreal engine", "unity", "mod", "retro", "emulation", "vr"],
|
|
"diy": ["diy", "woodworking", "3d printing", "cnc", "arduino", "raspberry pi",
|
|
"home automation", "solar", "electronics", "welding", "maker"],
|
|
"guides": ["how to", "tutorial", "guide", "beginner", "learn", "setup",
|
|
"install", "configure", "walkthrough", "step by step", "tips"],
|
|
}
|
|
|
|
|
|
def discover_trends() -> list[dict]:
|
|
"""Discover trending topics across all sources and score them."""
|
|
log.info("Starting trend discovery...")
|
|
topics = []
|
|
|
|
# Phase 1: Gather raw topics from web
|
|
raw_topics = _gather_web_topics()
|
|
|
|
# Phase 2: Use LLM to expand with seasonal/evergreen/FAQ topics
|
|
seasonal = _generate_seasonal_topics()
|
|
raw_topics.extend(seasonal)
|
|
|
|
# Phase 3: Score and assign verticals
|
|
scored = _score_and_assign(raw_topics)
|
|
|
|
# Phase 4: Deduplicate and rank
|
|
topics = _deduplicate_and_rank(scored)
|
|
|
|
# Phase 5: Store in DB
|
|
db = init_db()
|
|
for t in topics[:25]: # Top 25 topics per run
|
|
try:
|
|
db.execute("""
|
|
INSERT OR IGNORE INTO topics (title, vertical, trend_score, search_volume,
|
|
competition_score, freshness_score, evergreen_score, composite_score, sources)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (t["title"], t["vertical"], t["trend_score"], t.get("search_volume", 0),
|
|
t.get("competition_score", 0), t.get("freshness_score", 0),
|
|
t.get("evergreen_score", 0), t["composite_score"], json.dumps(t.get("sources", []))))
|
|
except Exception as e:
|
|
log.warning(f"Failed to insert topic {t['title']}: {e}")
|
|
db.commit()
|
|
db.close()
|
|
|
|
log.info(f"Discovered {len(topics)} topics, stored top 25")
|
|
return topics
|
|
|
|
|
|
def _gather_web_topics() -> list[str]:
|
|
"""Scrape trending topics from web sources."""
|
|
topics = set()
|
|
headers = {"User-Agent": "AutoPublisher/1.0"}
|
|
|
|
# Hacker News top stories
|
|
try:
|
|
r = requests.get(TREND_SOURCES[0]["url"], timeout=10)
|
|
if r.status_code == 200:
|
|
story_ids = r.json()[:20]
|
|
for sid in story_ids[:10]:
|
|
try:
|
|
sr = requests.get(
|
|
f"https://hacker-news.firebaseio.com/v0/item/{sid}.json",
|
|
timeout=5)
|
|
if sr.status_code == 200:
|
|
title = sr.json().get("title", "")
|
|
if title and len(title) > 10:
|
|
topics.add(title)
|
|
except Exception:
|
|
pass
|
|
except Exception as e:
|
|
log.warning(f"HN fetch failed: {e}")
|
|
|
|
# Reddit r/programming
|
|
try:
|
|
r = requests.get(TREND_SOURCES[1]["url"], headers=headers, timeout=10)
|
|
if r.status_code == 200:
|
|
for post in r.json()["data"]["children"][:15]:
|
|
title = post["data"].get("title", "")
|
|
if title:
|
|
topics.add(title)
|
|
except Exception as e:
|
|
log.warning(f"Reddit fetch failed: {e}")
|
|
|
|
# GitHub trending (last 7 days)
|
|
try:
|
|
since = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
|
|
url = TREND_SOURCES[2]["url"].format(since)
|
|
r = requests.get(url, headers=headers, timeout=10)
|
|
if r.status_code == 200:
|
|
for repo in r.json().get("items", [])[:10]:
|
|
desc = repo.get("description", "")
|
|
name = repo.get("full_name", "")
|
|
if desc:
|
|
topics.add(f"{name}: {desc}")
|
|
except Exception as e:
|
|
log.warning(f"GitHub fetch failed: {e}")
|
|
|
|
# arXiv AI papers
|
|
try:
|
|
import xml.etree.ElementTree as ET
|
|
r = requests.get(TREND_SOURCES[3]["url"], timeout=15)
|
|
if r.status_code == 200:
|
|
root = ET.fromstring(r.text)
|
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
|
for entry in root.findall("atom:entry", ns)[:10]:
|
|
title = entry.find("atom:title", ns)
|
|
if title is not None and title.text:
|
|
topics.add(title.text.strip().replace("\n", " "))
|
|
except Exception as e:
|
|
log.warning(f"arXiv fetch failed: {e}")
|
|
|
|
return list(topics)
|
|
|
|
|
|
def _generate_seasonal_topics() -> list[str]:
|
|
"""Use LLM to generate seasonal and evergreen topic suggestions."""
|
|
now = datetime.now()
|
|
month = now.strftime("%B")
|
|
prompt = f"""Generate 30 evergreen and seasonal content topics for {month} {now.year}.
|
|
These should be useful, educational articles people are searching for right now.
|
|
Cover these verticals: AI/ML, general tech, science, cryptocurrency, Linux, gaming, DIY/maker, and practical guides.
|
|
Respond with a JSON array of strings, each a compelling article title."""
|
|
|
|
try:
|
|
result = ollama_json(prompt, model="qwen3.5:4b", temperature=0.8)
|
|
if isinstance(result, list):
|
|
return result
|
|
return list(result.values())[0] if result else []
|
|
except Exception as e:
|
|
log.warning(f"Seasonal topic generation failed: {e}")
|
|
return []
|
|
|
|
|
|
def _score_and_assign(raw_topics: list[str]) -> list[dict]:
|
|
"""Score topics and assign to verticals using LLM, boosted by learning data."""
|
|
if not raw_topics:
|
|
return []
|
|
|
|
# Phase 0: Get learning insights from live sites
|
|
learning_insights = _get_learning_insights()
|
|
|
|
# Deduplicate first
|
|
unique = list(dict.fromkeys(raw_topics))[:50]
|
|
|
|
insights_text = ""
|
|
if learning_insights:
|
|
insights_text = f"\n\nLEARNING DATA — content that performs well on our sites:\n{json.dumps(learning_insights, indent=2)}\n\nUse this to boost composite_score for topics similar to what our audience already reads. Topics matching high-performing patterns get +10 to composite_score."
|
|
|
|
prompt = f"""You are a content strategist. Score and categorize these {len(unique)} topics.{insights_text}
|
|
|
|
Topics:
|
|
{json.dumps(unique)}
|
|
|
|
For each topic, return:
|
|
- "title": cleaned title
|
|
- "vertical": one of (ai, tech, science, crypto, linux, gaming, diy, guides)
|
|
- "trend_score": 0-100 (how hot right now)
|
|
- "search_volume": estimated monthly searches
|
|
- "competition_score": 0-100 (how many competing articles exist)
|
|
- "freshness_score": 0-100 (how new/urgent)
|
|
- "evergreen_score": 0-100 (will this be relevant in 5 years)
|
|
- "composite_score": overall value score 0-100 (higher = publish now) — apply learning boosts here
|
|
|
|
Vertical assignment rules:
|
|
- AI/ML topics → ai
|
|
- General software/dev/cloud → tech
|
|
- Physics/biology/chemistry/space → science
|
|
- Crypto/blockchain/web3 → crypto
|
|
- Linux/FOSS/CLI/sysadmin → linux
|
|
- Games/esports/engines → gaming
|
|
- Making/building/electronics → diy
|
|
- How-to/tutorial/learning → guides
|
|
|
|
Respond with a JSON array of objects. No markdown, no explanation."""
|
|
|
|
try:
|
|
result = ollama_json(prompt, model="qwen3.5:4b", temperature=0.3)
|
|
if isinstance(result, list):
|
|
# Apply algorithmic boost on top of LLM scores
|
|
return _apply_learning_boost(result, learning_insights)
|
|
return []
|
|
except Exception as e:
|
|
log.warning(f"Topic scoring failed: {e}")
|
|
return []
|
|
|
|
|
|
def _get_learning_insights() -> dict:
|
|
"""Query all 8 live sites for their top-performing content patterns."""
|
|
insights = {}
|
|
for vertical, vinfo in VERTICALS.items():
|
|
ct_ip = vinfo.get("ip")
|
|
if not ct_ip:
|
|
continue
|
|
try:
|
|
r = requests.get(f"http://{ct_ip}:5000/api/stats", timeout=5)
|
|
if r.status_code == 200:
|
|
data = r.json()
|
|
popular = data.get("popular", [])
|
|
if popular:
|
|
# Extract keyword patterns from popular articles
|
|
all_keywords = []
|
|
for art in popular:
|
|
kw_str = art.get("keywords", "[]")
|
|
try:
|
|
kws = json.loads(kw_str) if isinstance(kw_str, str) else kw_str
|
|
all_keywords.extend(kws)
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
|
|
# Most frequent keywords = winning topics
|
|
from collections import Counter
|
|
kw_counts = Counter(all_keywords)
|
|
|
|
insights[vertical] = {
|
|
"top_keywords": [kw for kw, _ in kw_counts.most_common(8)],
|
|
"top_articles": [a.get("title", "")[:80] for a in popular[:3]],
|
|
"avg_word_count": sum(a.get("word_count", 0) for a in popular) // max(len(popular), 1),
|
|
"total_articles": data.get("total_articles", 0),
|
|
}
|
|
except Exception:
|
|
pass
|
|
|
|
# Also check orchestrator's own performance_learning DB
|
|
try:
|
|
db = sqlite3.connect(str(DB_PATH))
|
|
db.row_factory = sqlite3.Row
|
|
for vertical in VERTICALS:
|
|
row = db.execute(
|
|
"SELECT * FROM performance_learning WHERE vertical=? ORDER BY updated_at DESC LIMIT 1",
|
|
(vertical,)
|
|
).fetchone()
|
|
if row:
|
|
if vertical not in insights:
|
|
insights[vertical] = {}
|
|
try:
|
|
insights[vertical]["stored_patterns"] = json.loads(row["top_patterns"])
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
db.close()
|
|
except Exception:
|
|
pass
|
|
|
|
return insights if insights else {}
|
|
|
|
|
|
def _apply_learning_boost(scored: list[dict], insights: dict) -> list[dict]:
|
|
"""Boost composite scores for topics matching winning patterns."""
|
|
if not insights:
|
|
return scored
|
|
|
|
for topic in scored:
|
|
vertical = topic.get("vertical", "")
|
|
title = topic.get("title", "").lower()
|
|
vin = insights.get(vertical, {})
|
|
top_kws = vin.get("top_keywords", [])
|
|
|
|
# Count keyword matches between topic title and winning keywords
|
|
matches = sum(1 for kw in top_kws if kw.lower() in title)
|
|
boost = min(matches * 8, 25) # Up to 25-point boost
|
|
|
|
# Boost for matching the optimal word count range (signals topic depth fits)
|
|
if vin.get("avg_word_count", 0) > 0:
|
|
boost += 3 # Minor boost for having any data
|
|
|
|
if boost > 0:
|
|
old_score = topic.get("composite_score", 50)
|
|
topic["composite_score"] = min(old_score + boost, 100)
|
|
topic["learning_boost"] = boost
|
|
|
|
return scored
|
|
|
|
|
|
def _deduplicate_and_rank(scored: list[dict]) -> list[dict]:
|
|
"""Remove near-duplicates and rank by composite score."""
|
|
seen_titles = set()
|
|
deduped = []
|
|
for t in sorted(scored, key=lambda x: x.get("composite_score", 0), reverse=True):
|
|
title_lower = t["title"].lower().strip()
|
|
# Check for near-duplicates
|
|
is_dup = False
|
|
for seen in seen_titles:
|
|
if title_lower in seen or seen in title_lower:
|
|
is_dup = True
|
|
break
|
|
if not is_dup:
|
|
seen_titles.add(title_lower)
|
|
deduped.append(t)
|
|
return deduped
|
|
|
|
|
|
# ─── Research ───────────────────────────────────────────────────────
|
|
def research_topic(topic_id: int, topic_title: str, vertical: str) -> dict:
|
|
"""Research a topic and build a knowledge package."""
|
|
log.info(f"Researching topic #{topic_id}: {topic_title}")
|
|
|
|
# Phase 1: Web search for sources
|
|
sources = _web_search_sources(topic_title)
|
|
|
|
# Phase 2: LLM deep research using ornith on GamingPC
|
|
research_prompt = f"""You are an expert researcher. Deeply research this topic:
|
|
|
|
TOPIC: {topic_title}
|
|
VERTICAL: {vertical}
|
|
|
|
SOURCES FOUND:
|
|
{json.dumps(sources[:5], indent=2)}
|
|
|
|
Extract and return as JSON:
|
|
{{
|
|
"facts": ["key fact 1", "key fact 2", ...], // 8-15 verified facts
|
|
"stats": [{{"stat": "...", "source": "..."}}, ...], // 3-8 statistics with sources
|
|
"definitions": [{{"term": "...", "definition": "..."}}, ...], // key terms
|
|
"faqs": [{{"question": "...", "answer": "..."}}, ...], // 5-10 FAQs
|
|
"misconceptions": ["common wrong belief", ...], // 3-5 corrections
|
|
"timeline": [{{"date": "...", "event": "..."}}, ...], // if applicable
|
|
"citations": [{{"text": "...", "source": "..."}}, ...], // sources
|
|
"examples": ["practical example 1", ...], // 3-5 real examples
|
|
"related_concepts": ["related topic 1", ...], // 5-10 related topics
|
|
"expertise_level": "beginner|intermediate|advanced",
|
|
"summary": "one paragraph comprehensive summary"
|
|
}}
|
|
|
|
Be accurate. Cite real sources. No hallucinations. Respond with ONLY valid JSON."""
|
|
|
|
try:
|
|
result = ollama_json(research_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
|
system="You are an expert research analyst. You produce accurate, well-cited research. Never fabricate information.")
|
|
except Exception as e:
|
|
log.error(f"Research LLM failed: {e}. Falling back to MacBook.")
|
|
result = ollama_json(research_prompt, model="qwen3.5:4b",
|
|
system="You are an expert research analyst. Be accurate and honest.")
|
|
|
|
# Store knowledge package
|
|
db = init_db()
|
|
db.execute("""
|
|
INSERT OR REPLACE INTO knowledge_packages (topic_id, facts, stats, definitions, faqs,
|
|
misconceptions, timeline, citations, examples, related_concepts, raw_sources)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
topic_id,
|
|
json.dumps(result.get("facts", [])),
|
|
json.dumps(result.get("stats", [])),
|
|
json.dumps(result.get("definitions", [])),
|
|
json.dumps(result.get("faqs", [])),
|
|
json.dumps(result.get("misconceptions", [])),
|
|
json.dumps(result.get("timeline", [])),
|
|
json.dumps(result.get("citations", [])),
|
|
json.dumps(result.get("examples", [])),
|
|
json.dumps(result.get("related_concepts", [])),
|
|
json.dumps(sources),
|
|
))
|
|
db.execute("UPDATE topics SET knowledge_package_id = ?, status = 'researched' WHERE id = ?",
|
|
(db.execute("SELECT last_insert_rowid()").fetchone()[0], topic_id))
|
|
db.commit()
|
|
db.close()
|
|
|
|
log.info(f"Research complete for topic #{topic_id}")
|
|
return result
|
|
|
|
|
|
def _web_search_sources(topic: str) -> list[dict]:
|
|
"""Search the web for topic sources."""
|
|
sources = []
|
|
# Try SearXNG first (self-hosted)
|
|
try:
|
|
r = requests.get("http://10.30.20.89:8888/search",
|
|
params={"q": topic, "format": "json", "categories": "general"},
|
|
timeout=10)
|
|
if r.status_code == 200:
|
|
for result in r.json().get("results", [])[:8]:
|
|
sources.append({
|
|
"title": result.get("title", ""),
|
|
"url": result.get("url", ""),
|
|
"snippet": result.get("content", "")[:200],
|
|
})
|
|
except Exception:
|
|
pass
|
|
|
|
return sources
|
|
|
|
|
|
# ─── Writing Pipeline ──────────────────────────────────────────────
|
|
def write_article(topic_id: int, topic_title: str, vertical: str,
|
|
knowledge_package: dict) -> dict:
|
|
"""Multi-agent writing pipeline: outline → draft → SEO → edit → fact-check."""
|
|
log.info(f"Writing article for topic #{topic_id}: {topic_title}")
|
|
|
|
kp_json = json.dumps(knowledge_package, indent=2)
|
|
|
|
# Agent 1: Outline
|
|
outline_prompt = f"""Create a detailed article outline for:
|
|
|
|
TITLE: {topic_title}
|
|
VERTICAL: {vertical}
|
|
|
|
KNOWLEDGE PACKAGE:
|
|
{kp_json}
|
|
|
|
Generate an outline with:
|
|
- Introduction hook
|
|
- 5-8 major sections with subsections
|
|
- Key takeaways
|
|
- FAQ section topics
|
|
- Call-to-action
|
|
|
|
Respond with JSON:
|
|
{{"sections": [{{"heading": "...", "subsections": ["..."]}}, ...], "faq_questions": ["..."], "cta": "..."}}"""
|
|
|
|
outline = ollama_json(outline_prompt, model="qwen3.5:4b", temperature=0.5)
|
|
|
|
# Agent 2: Technical Writer (ornith for quality)
|
|
draft_prompt = f"""Write a comprehensive, authoritative article.
|
|
|
|
TITLE: {topic_title}
|
|
VERTICAL: {vertical}
|
|
OUTLINE: {json.dumps(outline)}
|
|
FACTS: {json.dumps(knowledge_package.get('facts', []))}
|
|
STATS: {json.dumps(knowledge_package.get('stats', []))}
|
|
DEFINITIONS: {json.dumps(knowledge_package.get('definitions', []))}
|
|
EXAMPLES: {json.dumps(knowledge_package.get('examples', []))}
|
|
CITATIONS: {json.dumps(knowledge_package.get('citations', []))}
|
|
|
|
Write the full article in clean Markdown. Include:
|
|
- Engaging introduction that hooks the reader
|
|
- Well-structured sections following the outline
|
|
- Code blocks where relevant (for tech/linux)
|
|
- Pull quotes from key stats
|
|
- "Key Takeaway" boxes (use > blockquotes)
|
|
- FAQ section at the end
|
|
- Sources/citations section
|
|
|
|
Target: 1500-3000 words. Use a clear, authoritative but conversational tone.
|
|
DO NOT use AI clichés ("delve", "unleash", "game-changer", "in today's world").
|
|
Write like an expert explaining to an intelligent peer.
|
|
|
|
Respond with the FULL Markdown article. No JSON wrapper."""
|
|
|
|
draft = ollama_chat(draft_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
|
system="You are an expert technical writer. Write clear, accurate, engaging content. No AI clichés. No fluff.",
|
|
temperature=0.7, max_tokens=8192)
|
|
|
|
# Agent 3: Copy Editor (qwen, fast)
|
|
edit_prompt = f"""Edit and improve this article. Fix:
|
|
- Grammar and spelling
|
|
- Awkward phrasing
|
|
- Repetition
|
|
- Clarity issues
|
|
- Add transitions between sections
|
|
- Ensure consistent tone
|
|
- Break up overly long paragraphs
|
|
|
|
ARTICLE:
|
|
{draft}
|
|
|
|
Return the edited article in full Markdown. No JSON wrapper."""
|
|
|
|
edited = ollama_chat(edit_prompt, model="qwen3.5:4b", temperature=0.3, max_tokens=8192)
|
|
|
|
# Agent 4: SEO Optimization
|
|
seo_prompt = f"""Optimize this article for SEO. Generate:
|
|
|
|
1. SEO title (55-65 chars, include primary keyword)
|
|
2. Meta description (150-160 chars, compelling)
|
|
3. Suggested internal links (related topics from same vertical)
|
|
4. Tags/keywords (5-10)
|
|
|
|
ARTICLE TITLE: {topic_title}
|
|
VERTICAL: {vertical}
|
|
FIRST 500 CHARS: {edited[:500]}
|
|
|
|
Respond with JSON:
|
|
{{"seo_title": "...", "seo_description": "...", "keywords": ["..."], "internal_links": [{{"text": "...", "slug": "..."}}]}}"""
|
|
|
|
seo = ollama_json(seo_prompt, model="qwen3.5:4b", temperature=0.3)
|
|
|
|
# Agent 5: Fact Check (ornith)
|
|
factcheck_prompt = f"""Fact check this article. Verify:
|
|
1. Are the statistics accurate and properly sourced?
|
|
2. Are any claims unsubstantiated?
|
|
3. Are technical details correct?
|
|
4. Are dates and timelines accurate?
|
|
5. Is anything overstated or misleading?
|
|
|
|
ARTICLE:
|
|
{edited[:4000]}
|
|
|
|
FACTS USED:
|
|
{json.dumps(knowledge_package.get('facts', []))}
|
|
|
|
Respond with JSON:
|
|
{{"passed": true/false, "issues": ["issue 1", ...], "corrections": [{{"original": "...", "corrected": "..."}}]}}"""
|
|
|
|
factcheck = ollama_json(factcheck_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
|
system="You are a strict fact-checker. Flag everything questionable. Be conservative — if unsure, flag it.",
|
|
temperature=0.1)
|
|
|
|
# If fact check found issues, apply corrections
|
|
if not factcheck.get("passed", True):
|
|
corrections = factcheck.get("corrections", [])
|
|
if corrections:
|
|
fix_prompt = f"""Apply these corrections to the article:
|
|
|
|
{json.dumps(corrections, indent=2)}
|
|
|
|
ARTICLE:
|
|
{edited}
|
|
|
|
Return the corrected article in full Markdown. No JSON wrapper."""
|
|
edited = ollama_chat(fix_prompt, model="qwen3.5:4b", temperature=0.2)
|
|
|
|
# Calculate stats
|
|
word_count = len(edited.split())
|
|
reading_time = max(1, word_count // 200)
|
|
slug = topic_title.lower().strip()[:80]
|
|
slug = "".join(c if c.isalnum() or c in "- " else "" for c in slug)
|
|
slug = slug.replace(" ", "-").strip("-")
|
|
|
|
# Store article
|
|
db = init_db()
|
|
db.execute("""
|
|
INSERT OR REPLACE INTO articles (topic_id, vertical, title, slug, content_md,
|
|
seo_title, seo_description, word_count, reading_time_minutes, status)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft')
|
|
""", (topic_id, vertical, topic_title, slug, edited,
|
|
seo.get("seo_title", topic_title[:65]),
|
|
seo.get("seo_description", ""),
|
|
word_count, reading_time))
|
|
db.execute("UPDATE topics SET status = 'written', article_id = last_insert_rowid() WHERE id = ?",
|
|
(topic_id,))
|
|
db.commit()
|
|
db.close()
|
|
|
|
log.info(f"Article written for #{topic_id}: {word_count} words, {reading_time}min read")
|
|
return {
|
|
"topic_id": topic_id,
|
|
"title": topic_title,
|
|
"slug": slug,
|
|
"content": edited,
|
|
"seo": seo,
|
|
"word_count": word_count,
|
|
"reading_time": reading_time,
|
|
"factcheck_passed": factcheck.get("passed", True),
|
|
}
|
|
|
|
|
|
# ─── Site Builder ──────────────────────────────────────────────────
|
|
def build_site(vertical: str, articles: list[dict]) -> str:
|
|
"""Build static HTML site from articles."""
|
|
log.info(f"Building site for {vertical}...")
|
|
|
|
site_dir = BASE_DIR / "sites" / vertical
|
|
site_dir.mkdir(parents=True, exist_ok=True)
|
|
(site_dir / "articles").mkdir(exist_ok=True)
|
|
(site_dir / "assets" / "images").mkdir(parents=True, exist_ok=True)
|
|
|
|
# Generate each article page
|
|
for article in articles:
|
|
html = _article_to_html(article, vertical)
|
|
article_path = site_dir / "articles" / f"{article['slug']}.html"
|
|
article_path.write_text(html)
|
|
|
|
# Generate homepage
|
|
homepage = _build_homepage(vertical, articles)
|
|
(site_dir / "index.html").write_text(homepage)
|
|
|
|
# Generate RSS feed
|
|
rss = _build_rss(vertical, articles)
|
|
(site_dir / "rss.xml").write_text(rss)
|
|
|
|
# Generate sitemap
|
|
sitemap = _build_sitemap(vertical, articles)
|
|
(site_dir / "sitemap.xml").write_text(sitemap)
|
|
|
|
log.info(f"Site built for {vertical}: {len(articles)} articles")
|
|
return str(site_dir)
|
|
|
|
|
|
def _article_to_html(article: dict, vertical: str) -> str:
|
|
"""Convert markdown article to full HTML page."""
|
|
# Simple markdown-to-HTML conversion (rudimentary but functional)
|
|
content = article.get("content_md", article.get("content", ""))
|
|
html_body = _md_to_html(content)
|
|
|
|
seo_title = article.get("seo_title", article.get("title", ""))
|
|
seo_desc = article.get("seo_description", "")
|
|
og_image = article.get("og_image", f"/assets/images/{vertical}-default.webp")
|
|
json_ld = _build_json_ld(article, vertical)
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{seo_title}</title>
|
|
<meta name="description" content="{seo_desc}">
|
|
<meta property="og:title" content="{seo_title}">
|
|
<meta property="og:description" content="{seo_desc}">
|
|
<meta property="og:image" content="{og_image}">
|
|
<meta property="og:type" content="article">
|
|
<meta name="twitter:card" content="summary_large_image">
|
|
<link rel="canonical" href="https://{vertical}.thetempleofdoom.com/articles/{article.get('slug', '')}">
|
|
<script type="application/ld+json">{json.dumps(json_ld)}</script>
|
|
<link rel="stylesheet" href="/assets/style.css">
|
|
<link href="/rss.xml" rel="alternate" type="application/rss+xml" title="{vertical}.thetempleofdoom.com">
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<nav>
|
|
<a href="/" class="logo">{vertical}.thetempleofdoom.com</a>
|
|
<div class="nav-links">
|
|
<a href="/">Home</a>
|
|
<a href="/articles/">Articles</a>
|
|
<a href="/about/">About</a>
|
|
</div>
|
|
</nav>
|
|
</header>
|
|
<main>
|
|
<article>
|
|
<header class="article-header">
|
|
<h1>{article.get("title", "")}</h1>
|
|
<div class="meta">
|
|
<time>{article.get("published_at", "")}</time>
|
|
<span>{article.get("reading_time_minutes", 5)} min read</span>
|
|
<span>{article.get("word_count", 0)} words</span>
|
|
</div>
|
|
</header>
|
|
<div class="article-content">
|
|
{html_body}
|
|
</div>
|
|
<footer class="article-footer">
|
|
<div class="tags">
|
|
{_build_tag_links(article.get("keywords", []), vertical)}
|
|
</div>
|
|
<div class="sources">
|
|
<h3>Sources</h3>
|
|
{_build_sources_html(article)}
|
|
</div>
|
|
</footer>
|
|
</article>
|
|
<aside class="related">
|
|
<h3>Related Articles</h3>
|
|
<!-- Dynamically populated by site builder -->
|
|
</aside>
|
|
</main>
|
|
<footer class="site-footer">
|
|
<p>© {datetime.now().year} {vertical}.thetempleofdoom.com — Built by AI, curated for humans.</p>
|
|
<nav>
|
|
<a href="/rss.xml">RSS</a>
|
|
<a href="/sitemap.xml">Sitemap</a>
|
|
<a href="/privacy/">Privacy</a>
|
|
</nav>
|
|
</footer>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def _md_to_html(md: str) -> str:
|
|
"""Basic markdown to HTML conversion."""
|
|
import re
|
|
lines = md.split("\n")
|
|
html = []
|
|
in_code_block = False
|
|
code_lines = []
|
|
code_lang = ""
|
|
|
|
i = 0
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
|
|
# Code blocks
|
|
if line.strip().startswith("```"):
|
|
if in_code_block:
|
|
code = "\n".join(code_lines)
|
|
html.append(f'<pre><code class="language-{code_lang}">{_escape_html(code)}</code></pre>')
|
|
code_lines = []
|
|
in_code_block = False
|
|
else:
|
|
in_code_block = True
|
|
code_lang = line.strip()[3:].strip()
|
|
i += 1
|
|
continue
|
|
|
|
if in_code_block:
|
|
code_lines.append(line)
|
|
i += 1
|
|
continue
|
|
|
|
# Headers
|
|
if line.startswith("### "):
|
|
html.append(f"<h3>{_inline_md(line[4:])}</h3>")
|
|
elif line.startswith("## "):
|
|
html.append(f"<h2>{_inline_md(line[3:])}</h2>")
|
|
elif line.startswith("# "):
|
|
html.append(f"<h1>{_inline_md(line[2:])}</h1>")
|
|
|
|
# Blockquotes
|
|
elif line.startswith("> "):
|
|
html.append(f'<blockquote><p>{_inline_md(line[2:])}</p></blockquote>')
|
|
|
|
# Lists
|
|
elif line.strip().startswith("- ") or line.strip().startswith("* "):
|
|
html.append(f"<li>{_inline_md(line.strip()[2:])}</li>")
|
|
elif re.match(r"^\d+\.", line.strip()):
|
|
text = re.sub(r"^\d+\.\s*", "", line.strip())
|
|
html.append(f"<li>{_inline_md(text)}</li>")
|
|
|
|
# Horizontal rule
|
|
elif line.strip() in ("---", "***", "___"):
|
|
html.append("<hr>")
|
|
|
|
# Empty line
|
|
elif not line.strip():
|
|
html.append("")
|
|
|
|
# Paragraph
|
|
else:
|
|
html.append(f"<p>{_inline_md(line)}</p>")
|
|
|
|
i += 1
|
|
|
|
return "\n".join(html)
|
|
|
|
|
|
def _inline_md(text: str) -> str:
|
|
"""Convert inline markdown to HTML."""
|
|
import re
|
|
# Bold
|
|
text = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", text)
|
|
# Italic
|
|
text = re.sub(r"\*(.+?)\*", r"<em>\1</em>", text)
|
|
# Inline code
|
|
text = re.sub(r"`(.+?)`", r"<code>\1</code>", text)
|
|
# Links
|
|
text = re.sub(r"\[(.+?)\]\((.+?)\)", r'<a href="\2">\1</a>', text)
|
|
return text
|
|
|
|
|
|
def _escape_html(text: str) -> str:
|
|
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
|
|
|
|
def _build_homepage(vertical: str, articles: list[dict]) -> str:
|
|
"""Build the homepage for a vertical site."""
|
|
articles_html = ""
|
|
for a in sorted(articles, key=lambda x: x.get("published_at", ""), reverse=True)[:20]:
|
|
articles_html += f"""
|
|
<article class="card">
|
|
<h2><a href="/articles/{a.get('slug', '')}.html">{a.get('title', '')}</a></h2>
|
|
<p class="meta">{a.get('published_at', '')} · {a.get('reading_time_minutes', 5)} min read</p>
|
|
<p>{a.get('seo_description', '')}</p>
|
|
</article>"""
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{vertical}.thetempleofdoom.com — {vertical.title()} articles, guides & insights</title>
|
|
<meta name="description" content="Expert {vertical} articles, tutorials, and insights. Updated daily with fresh, useful content.">
|
|
<link rel="stylesheet" href="/assets/style.css">
|
|
<link href="/rss.xml" rel="alternate" type="application/rss+xml" title="{vertical}.thetempleofdoom.com">
|
|
<script type="application/ld+json">{json.dumps(_site_json_ld(vertical))}</script>
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<nav>
|
|
<a href="/" class="logo">{vertical}.thetempleofdoom.com</a>
|
|
<div class="nav-links">
|
|
<a href="/">Home</a>
|
|
<a href="/articles/">Articles</a>
|
|
<a href="/about/">About</a>
|
|
<a href="/rss.xml" class="rss-link">RSS</a>
|
|
</div>
|
|
</nav>
|
|
</header>
|
|
<main>
|
|
<section class="hero">
|
|
<h1>{vertical.title()} Insights & Guides</h1>
|
|
<p>Expert articles, tutorials, and deep dives. Updated daily.</p>
|
|
</section>
|
|
<section class="articles-grid">
|
|
{articles_html}
|
|
</section>
|
|
</main>
|
|
<footer class="site-footer">
|
|
<p>© {datetime.now().year} {vertical}.thetempleofdoom.com</p>
|
|
<nav>
|
|
<a href="/rss.xml">RSS</a>
|
|
<a href="/sitemap.xml">Sitemap</a>
|
|
<a href="/privacy/">Privacy</a>
|
|
</nav>
|
|
</footer>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def _build_rss(vertical: str, articles: list[dict]) -> str:
|
|
"""Build RSS 2.0 feed."""
|
|
items = ""
|
|
domain = f"{vertical}.thetempleofdoom.com"
|
|
for a in sorted(articles, key=lambda x: x.get("published_at", ""), reverse=True)[:20]:
|
|
items += f"""
|
|
<item>
|
|
<title>{_escape_xml(a.get('title', ''))}</title>
|
|
<link>https://{domain}/articles/{a.get('slug', '')}.html</link>
|
|
<guid>https://{domain}/articles/{a.get('slug', '')}.html</guid>
|
|
<description>{_escape_xml(a.get('seo_description', ''))}</description>
|
|
<pubDate>{a.get('published_at', '')}</pubDate>
|
|
</item>"""
|
|
|
|
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
|
<channel>
|
|
<title>{vertical}.thetempleofdoom.com</title>
|
|
<link>https://{domain}</link>
|
|
<description>Expert {vertical} articles and insights</description>
|
|
<language>en-us</language>
|
|
<lastBuildDate>{datetime.now().isoformat()}</lastBuildDate>
|
|
<atom:link href="https://{domain}/rss.xml" rel="self" type="application/rss+xml"/>
|
|
{items}
|
|
</channel>
|
|
</rss>"""
|
|
|
|
|
|
def _build_sitemap(vertical: str, articles: list[dict]) -> str:
|
|
"""Build XML sitemap."""
|
|
urls = ""
|
|
domain = f"{vertical}.thetempleofdoom.com"
|
|
urls += f"""
|
|
<url>
|
|
<loc>https://{domain}/</loc>
|
|
<changefreq>daily</changefreq>
|
|
<priority>1.0</priority>
|
|
</url>"""
|
|
for a in articles:
|
|
urls += f"""
|
|
<url>
|
|
<loc>https://{domain}/articles/{a.get('slug', '')}.html</loc>
|
|
<lastmod>{a.get('published_at', datetime.now().strftime('%Y-%m-%d'))}</lastmod>
|
|
<changefreq>weekly</changefreq>
|
|
<priority>0.8</priority>
|
|
</url>"""
|
|
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{urls}
|
|
</urlset>"""
|
|
|
|
|
|
def _build_json_ld(article: dict, vertical: str) -> dict:
|
|
return {
|
|
"@context": "https://schema.org",
|
|
"@type": "Article",
|
|
"headline": article.get("title", ""),
|
|
"description": article.get("seo_description", ""),
|
|
"datePublished": article.get("published_at", ""),
|
|
"author": {"@type": "Organization", "name": f"{vertical}.thetempleofdoom.com"},
|
|
"publisher": {"@type": "Organization", "name": f"{vertical}.thetempleofdoom.com"},
|
|
}
|
|
|
|
|
|
def _site_json_ld(vertical: str) -> dict:
|
|
return {
|
|
"@context": "https://schema.org",
|
|
"@type": "WebSite",
|
|
"name": f"{vertical}.thetempleofdoom.com",
|
|
"url": f"https://{vertical}.thetempleofdoom.com",
|
|
"description": f"Expert {vertical} articles, tutorials, and insights. Updated daily.",
|
|
}
|
|
|
|
|
|
def _build_tag_links(keywords: list, vertical: str) -> str:
|
|
return " ".join(f'<a href="/tag/{k.lower().replace(" ", "-")}" class="tag">{k}</a>' for k in (keywords or []))
|
|
|
|
|
|
def _build_sources_html(article: dict) -> str:
|
|
# Extract sources from knowledge package if available
|
|
return "<p>Sources available in the original knowledge package.</p>"
|
|
|
|
|
|
def _escape_xml(text: str) -> str:
|
|
return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
|
|
|
|
|
# ─── Deploy ─────────────────────────────────────────────────────────
|
|
def deploy_site(vertical: str, site_dir: str, ct_ip: str) -> bool:
|
|
"""Deploy a site to its Proxmox CT."""
|
|
log.info(f"Deploying {vertical} to {ct_ip}...")
|
|
|
|
# Create deployment tarball
|
|
import tarfile
|
|
tarball = BASE_DIR / "core" / f"{vertical}_deploy.tar.gz"
|
|
with tarfile.open(tarball, "w:gz") as tar:
|
|
tar.add(site_dir, arcname=vertical)
|
|
|
|
try:
|
|
# Push to CT
|
|
result = os.system(f"scp {tarball} root@{ct_ip}:/tmp/ 2>/dev/null")
|
|
if result != 0:
|
|
log.error(f"scp failed for {vertical}")
|
|
return False
|
|
|
|
os.system(f"ssh root@{ct_ip} 'cd /tmp && tar xzf {vertical}_deploy.tar.gz && "
|
|
f"cp -r {vertical}/* /var/www/html/ && systemctl restart nginx' 2>/dev/null")
|
|
|
|
# Verify
|
|
time.sleep(2)
|
|
verify = os.popen(f"curl -s -o /dev/null -w '%{{http_code}}' http://{ct_ip}:80/").read().strip()
|
|
if verify == "200":
|
|
log.info(f"Deploy {vertical} successful: HTTP {verify}")
|
|
return True
|
|
else:
|
|
log.error(f"Deploy {vertical} verify failed: HTTP {verify}")
|
|
return False
|
|
except Exception as e:
|
|
log.error(f"Deploy {vertical} exception: {e}")
|
|
return False
|
|
finally:
|
|
if tarball.exists():
|
|
tarball.unlink()
|
|
|
|
|
|
# ─── Pipeline Runner ────────────────────────────────────────────────
|
|
def run_daily_pipeline(max_articles: int = 3):
|
|
"""Run the full daily publishing pipeline."""
|
|
log.info("=" * 60)
|
|
log.info("DAILY PUBLISHING PIPELINE STARTING")
|
|
log.info("=" * 60)
|
|
|
|
run_id = None
|
|
db = init_db()
|
|
try:
|
|
# Log the run
|
|
cur = db.execute("INSERT INTO pipeline_runs (run_type, started_at) VALUES ('daily', datetime('now'))")
|
|
run_id = cur.lastrowid
|
|
db.commit()
|
|
|
|
# Step 1: Discover trends
|
|
topics = discover_trends()
|
|
db.execute("UPDATE pipeline_runs SET topics_discovered = ? WHERE id = ?",
|
|
(len(topics), run_id))
|
|
db.commit()
|
|
|
|
if not topics:
|
|
log.warning("No topics discovered. Exiting.")
|
|
return
|
|
|
|
# Step 2: Get approved topics (composite_score > 60, not yet published)
|
|
top_topics = db.execute("""
|
|
SELECT id, title, vertical, composite_score FROM topics
|
|
WHERE status = 'discovered' AND composite_score > 60
|
|
ORDER BY composite_score DESC LIMIT ?
|
|
""", (max_articles,)).fetchall()
|
|
|
|
articles_published = 0
|
|
vertical_articles = {}
|
|
|
|
for topic_row in top_topics:
|
|
topic_id, topic_title, vertical, score = topic_row
|
|
|
|
# Step 3: Research
|
|
kp = research_topic(topic_id, topic_title, vertical)
|
|
|
|
# Step 4: Write
|
|
article = write_article(topic_id, topic_title, vertical, kp)
|
|
|
|
# Collect articles per vertical for site building
|
|
if vertical not in vertical_articles:
|
|
vertical_articles[vertical] = []
|
|
vertical_articles[vertical].append(article)
|
|
|
|
articles_published += 1
|
|
log.info(f" ✓ Published: {topic_title} → {vertical}")
|
|
|
|
# Step 5: Publish to live site APIs
|
|
for vertical, articles in vertical_articles.items():
|
|
vinfo = VERTICALS.get(vertical, {})
|
|
ct_ip = vinfo.get("ip")
|
|
if not ct_ip:
|
|
log.warning(f"No CT IP for {vertical} — skipping publish")
|
|
continue
|
|
|
|
api_url = f"http://{ct_ip}:5000/api/publish"
|
|
for article in articles:
|
|
try:
|
|
r = requests.post(api_url, json=article,
|
|
headers={"Authorization": "Bearer auto-publish-2026"},
|
|
timeout=15)
|
|
if r.status_code in (200, 201):
|
|
log.info(f" 📤 Published to {vertical}: {article.get('title', '')[:60]}")
|
|
else:
|
|
log.warning(f" ❌ {vertical} API returned {r.status_code}: {r.text[:100]}")
|
|
except Exception as e:
|
|
log.warning(f" ❌ Failed to publish to {vertical}: {e}")
|
|
|
|
# Update run log
|
|
db.execute("""
|
|
UPDATE pipeline_runs SET articles_published = ?, finished_at = datetime('now')
|
|
WHERE id = ?
|
|
""", (articles_published, run_id))
|
|
db.commit()
|
|
|
|
log.info(f"Pipeline complete: {articles_published} articles published across {len(vertical_articles)} sites")
|
|
|
|
except Exception as e:
|
|
log.error(f"Pipeline failed: {e}", exc_info=True)
|
|
if run_id:
|
|
db.execute("UPDATE pipeline_runs SET errors = ?, finished_at = datetime('now') WHERE id = ?",
|
|
(json.dumps([str(e)]), run_id))
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
# ─── CLI ────────────────────────────────────────────────────────────
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
ap = argparse.ArgumentParser(description="Autonomous Publishing System")
|
|
ap.add_argument("command", choices=["init", "discover", "research", "write", "build", "deploy", "run", "status"])
|
|
ap.add_argument("--topic-id", type=int, help="Topic ID for research/write")
|
|
ap.add_argument("--vertical", type=str, help="Vertical name")
|
|
ap.add_argument("--max", type=int, default=3, help="Max articles per run")
|
|
|
|
args = ap.parse_args()
|
|
|
|
if args.command == "init":
|
|
init_db()
|
|
print("✓ Database initialized")
|
|
|
|
elif args.command == "discover":
|
|
topics = discover_trends()
|
|
for t in topics[:15]:
|
|
print(f" [{t['vertical']}] {t['title'][:80]} (score: {t['composite_score']})")
|
|
|
|
elif args.command == "run":
|
|
run_daily_pipeline(max_articles=args.max)
|
|
|
|
elif args.command == "status":
|
|
db = init_db()
|
|
run = db.execute("SELECT * FROM pipeline_runs ORDER BY id DESC LIMIT 1").fetchone()
|
|
topics_count = db.execute("SELECT COUNT(*) FROM topics").fetchone()[0]
|
|
articles_count = db.execute("SELECT COUNT(*) FROM articles").fetchone()[0]
|
|
published = db.execute("SELECT COUNT(*) FROM articles WHERE status = 'published'").fetchone()[0]
|
|
print(f"Last run: {run}")
|
|
print(f"Topics in DB: {topics_count}")
|
|
print(f"Articles: {articles_count} ({published} published)")
|
|
|
|
else:
|
|
print(f"Unknown command: {args.command}")
|