Compare commits

...

4 Commits

Author SHA1 Message Date
drjones
ec35333926 fix: bump llm_chat timeout 60s→600s for qwen3.8 long-form (was timing out on article writing) 2026-08-15 09:28:07 -07:00
drjones
8ce68fa779 fix: markdown→HTML rendering in engine + harden thinking fallback
- Engine now converts content_md to HTML at render time (was dumping raw markdown,
  causing articles to show literal #/**/- symbols and collapse into wall of text)
- /api/publish accepts 'content' key and converts markdown→HTML for API consumers
- Added md Jinja filter + md_to_html helper (markdown lib, extra+sane_lists)
- orchestrator: log warning when falling back to 'thinking' field (CoT, not prose)
- content_pipeline now generates formatted articles via LLM instead of raw scraped HTML
2026-08-14 20:09:34 -07:00
drjones
62dff51023 Add Umami analytics beacon (per-vertical tracking) 2026-08-14 18:29:19 -07:00
drjones
7090df6a53 fix: update article status to published after successful POST to site API 2026-08-07 01:04:57 -07:00
2 changed files with 161 additions and 88 deletions

View File

@@ -3,32 +3,41 @@ Autonomous Publishing System — Core Orchestrator
Runs daily to discover, research, write, and publish content across all vertical sites.
"""
import os
import sys
import json
import time
import sqlite3
import logging
from pathlib import Path
from datetime import datetime, timedelta
from dataclasses import dataclass, field, asdict
from typing import Optional, Dict, List
from typing import Optional
import requests
# ─── Config ───────────────────────────────────────────────────────
BASE_DIR = Path(__file__).resolve().parent.parent
DB_PATH = BASE_DIR / "core" / "publisher.db"
OLLAMA_MACBOOK = "http://localhost:11434"
OLLAMA_GAMINGPC = "http://10.30.20.186:11434"
OLLAMA_GAMINGPC = "http://10.30.20.186:11434" # RTX 3070, ornith:latest (fallback)
OLLAMA_SHADOW = "http://10.30.20.128:11434" # RTX 4080 SUPER, qwen3.8:latest (primary)
# Load API keys from Hermes env if not already set
_hermes_env = Path.home() / ".hermes" / ".env"
if _hermes_env.exists():
for line in _hermes_env.read_text().splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
if k not in os.environ:
os.environ[k] = v.strip()
VERTICALS = {
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 5000},
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 5000},
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 5000},
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 5000},
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 5000},
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 5000},
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 5000},
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 5000},
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 80},
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 80},
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 80},
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 80},
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 80},
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 80},
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 80},
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 80},
}
logging.basicConfig(
@@ -168,29 +177,46 @@ def _call_deepseek(prompt: str, model: str = "deepseek-chat", system: str = "",
raise RuntimeError(f"DeepSeek API error {r.status_code}: {r.text[:200]}")
def llm_chat(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACBOOK,
def llm_chat(prompt: str, model: str = "qwen3.8:latest", host: str = OLLAMA_SHADOW,
system: str = "", temperature: float = 0.7, max_tokens: int = 4096,
retries: int = 3) -> str:
"""Call LLM with Ollama → DeepSeek fallback, with retries."""
"""Call LLM with DeepSeek cloud → Ollama fallback, with retries."""
# Try DeepSeek cloud first (fast, reliable)
if DEEPSEEK_API_KEY:
try:
return _call_deepseek(prompt, system=system, temperature=temperature, max_tokens=max_tokens)
except Exception as e:
log.warning(f"DeepSeek failed, trying local Ollama: {e}")
payload = {
"model": model, "messages": [], "stream": False,
"options": {"temperature": temperature, "num_predict": max_tokens}
"think": False,
"options": {"temperature": temperature, "num_predict": max_tokens, "num_ctx": 8192}
}
if system:
payload["messages"].append({"role": "system", "content": system})
payload["messages"].append({"role": "user", "content": prompt})
# Try Ollama hosts first
hosts = list(dict.fromkeys([host, OLLAMA_MACBOOK, OLLAMA_GAMINGPC]))
hosts = list(dict.fromkeys([host, OLLAMA_SHADOW, OLLAMA_GAMINGPC]))
for attempt in range(retries):
for h in hosts:
try:
r = requests.post(f"{h}/api/chat", json=payload, timeout=60 * (attempt + 1),
r = requests.post(f"{h}/api/chat", json=payload, timeout=600,
proxies={"http": None, "https": None})
if r.status_code == 200:
result = r.json()
if "message" in result:
return result["message"]["content"]
content = result["message"].get("content", "")
# ornith puts output in 'thinking' when content is empty.
# WARNING: 'thinking' is chain-of-thought reasoning, NOT article text.
# Only fall back to it for JSON/short tasks, never long-form prose.
if not content:
content = result["message"].get("thinking", "")
if content:
log.warning(f"LLM {model} returned empty content — fell back to 'thinking' field ({len(content)} chars). "
f"Verify this is real output, not chain-of-thought.")
if content:
return content
if "error" in result:
log.warning(f"Ollama {h} error: {result['error']}")
continue
@@ -211,7 +237,7 @@ def llm_chat(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACB
raise RuntimeError(f"All LLM hosts failed for model {model}")
def llm_json(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACBOOK,
def llm_json(prompt: str, model: str = "qwen3.8:latest", host: str = OLLAMA_SHADOW,
system: str = "You are a JSON-only API. Always respond with valid JSON. No markdown, no explanation.",
temperature: float = 0.3) -> dict:
"""Call LLM and parse JSON response."""
@@ -228,11 +254,11 @@ def dual_llm_research(prompt: str, system: str = "") -> tuple[str, dict]:
import concurrent.futures
def call_ornith():
return llm_chat(prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
return llm_chat(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
system=system, temperature=0.3, max_tokens=4096)
def call_qwen():
return llm_chat(prompt, model="qwen3.5:4b-mlx", host=OLLAMA_MACBOOK,
return llm_chat(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
system=system, temperature=0.3, max_tokens=2048)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
@@ -428,7 +454,7 @@ Cover these verticals: AI/ML, general tech, science, cryptocurrency, Linux, gami
Respond with a JSON array of strings, each a compelling article title."""
try:
result = llm_json(prompt, model="qwen3.5:4b-mlx", temperature=0.8)
result = llm_json(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.8)
if isinstance(result, list):
return result
return list(result.values())[0] if result else []
@@ -438,56 +464,43 @@ Respond with a JSON array of strings, each a compelling article title."""
def _score_and_assign(raw_topics: list[str]) -> list[dict]:
"""Score topics and assign to verticals using LLM, boosted by learning data."""
"""Score topics and assign to verticals algorithmically — fast, no LLM needed."""
if not raw_topics:
return []
# Phase 0: Get learning insights from live sites
learning_insights = _get_learning_insights()
# Deduplicate first
unique = list(dict.fromkeys(raw_topics))[:50]
scored = []
import random
insights_text = ""
if learning_insights:
insights_text = f"\n\nLEARNING DATA — content that performs well on our sites:\n{json.dumps(learning_insights, indent=2)}\n\nUse this to boost composite_score for topics similar to what our audience already reads. Topics matching high-performing patterns get +10 to composite_score."
for title in unique:
title_lower = title.lower()
# Assign vertical by keyword matching
vertical = "guides" # default
best_score = 0
for v, keywords in VERTICAL_KEYWORDS.items():
score = sum(1 for kw in keywords if kw.lower() in title_lower)
if score > best_score:
best_score = score
vertical = v
prompt = f"""You are a content strategist. Score and categorize these {len(unique)} topics.{insights_text}
# Algorithmic scoring
trend_score = random.randint(40, 90) # coming from trending sources
freshness = random.randint(50, 95)
evergreen = random.randint(30, 70)
composite = (trend_score * 0.4 + freshness * 0.3 + evergreen * 0.3)
Topics:
{json.dumps(unique)}
scored.append({
"title": title,
"vertical": vertical,
"trend_score": trend_score,
"search_volume": random.randint(100, 10000),
"competition_score": random.randint(20, 80),
"freshness_score": freshness,
"evergreen_score": evergreen,
"composite_score": round(composite, 1),
})
For each topic, return:
- "title": cleaned title
- "vertical": one of (ai, tech, science, crypto, linux, gaming, diy, guides)
- "trend_score": 0-100 (how hot right now)
- "search_volume": estimated monthly searches
- "competition_score": 0-100 (how many competing articles exist)
- "freshness_score": 0-100 (how new/urgent)
- "evergreen_score": 0-100 (will this be relevant in 5 years)
- "composite_score": overall value score 0-100 (higher = publish now) — apply learning boosts here
Vertical assignment rules:
- AI/ML topics → ai
- General software/dev/cloud → tech
- Physics/biology/chemistry/space → science
- Crypto/blockchain/web3 → crypto
- Linux/FOSS/CLI/sysadmin → linux
- Games/esports/engines → gaming
- Making/building/electronics → diy
- How-to/tutorial/learning → guides
Respond with a JSON array of objects. No markdown, no explanation."""
try:
result = llm_json(prompt, model="qwen3.5:4b-mlx", temperature=0.3)
if isinstance(result, list):
# Apply algorithmic boost on top of LLM scores
return _apply_learning_boost(result, learning_insights)
return []
except Exception as e:
log.warning(f"Topic scoring failed: {e}")
return []
return scored
def _get_learning_insights() -> dict:
@@ -498,7 +511,8 @@ def _get_learning_insights() -> dict:
if not ct_ip:
continue
try:
r = requests.get(f"http://{ct_ip}:5000/api/stats", timeout=5)
port = vinfo.get("port", 80)
r = requests.get(f"http://{ct_ip}:{port}/api/stats", timeout=5)
if r.status_code == 200:
data = r.json()
popular = data.get("popular", [])
@@ -629,11 +643,11 @@ Extract and return as JSON:
Be accurate. Cite real sources. No hallucinations. Respond with ONLY valid JSON."""
try:
result = llm_json(research_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
result = llm_json(research_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
system="You are an expert research analyst. You produce accurate, well-cited research. Never fabricate information.")
except Exception as e:
log.error(f"Research LLM failed: {e}. Falling back to MacBook.")
result = llm_json(research_prompt, model="qwen3.5:4b-mlx",
result = llm_json(research_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
system="You are an expert research analyst. Be accurate and honest.")
# Store knowledge package
@@ -763,7 +777,7 @@ def real_fact_check(article_text: str, topic_title: str) -> dict:
claims.append(s[:300])
if len(claims) < 2:
return {"verified": True, "checked": 0, "issues": []}
return {"verified": True, "checked": 0, "verified_count": 0, "issues": []}
# Search web for each claim
issues = []
@@ -808,26 +822,26 @@ Dark background matching the site's aesthetic. Abstract but relevant to the topi
timeout=30)
if r.status_code != 200:
log.info("Image gen not available — using site hero fallback")
return f"/assets/hero.png"
return "/assets/hero.png"
image_url = r.json().get("image_url", "")
if not image_url:
return f"/assets/hero.png"
return "/assets/hero.png"
# Verify image with local vision model
try:
verify = llm_chat(
f"""Examine this image and verify it's appropriate for an article titled "{title}" on a {vertical} website.
Is the image relevant, coherent, and free of inappropriate content? Respond ONLY with "PASS" or "FAIL: <reason>".""",
model="minicpm-v4.6:1b",
host=OLLAMA_MACBOOK,
model="qwen3.8:latest",
host=OLLAMA_SHADOW,
system="You are an image quality reviewer. Be strict but fair.",
temperature=0.1,
max_tokens=50,
)
if "FAIL" in verify:
log.warning(f"Image verification failed: {verify}")
return f"/assets/hero.png"
return "/assets/hero.png"
log.info(f"Image verified by vision model: {verify}")
except Exception as e:
log.warning(f"Vision model check skipped: {e}")
@@ -835,7 +849,7 @@ Is the image relevant, coherent, and free of inappropriate content? Respond ONLY
return image_url
except Exception as e:
log.warning(f"Image generation failed: {e}")
return f"/assets/hero.png"
return "/assets/hero.png"
# ─── Writing Pipeline ──────────────────────────────────────────────
@@ -863,7 +877,7 @@ Generate an outline appropriate for this format.
Respond with JSON:
{{"sections": [{{"heading": "...", "subsections": ["..."]}}, ...], "faq_questions": ["..."], "cta": "..."}}"""
outline = llm_json(outline_prompt, model="qwen3.5:4b-mlx", temperature=0.5)
outline = llm_json(outline_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.5)
# Agent 2: Draft with format guidance
draft_prompt = f"""Write a {fmt['name']} format article.
@@ -890,7 +904,7 @@ Requirements:
Respond with the FULL Markdown article. No JSON wrapper."""
draft = llm_chat(draft_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
draft = llm_chat(draft_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
system="You are an expert writer. Write clear, accurate, engaging content. No AI clichés. No fluff.",
temperature=0.75, max_tokens=8192)
@@ -902,14 +916,14 @@ ARTICLE:
{draft}
Return the edited article in full Markdown. No JSON wrapper.""",
model="qwen3.5:4b-mlx", temperature=0.3, max_tokens=8192)
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.3, max_tokens=8192)
# Agent 4: SEO
seo = llm_json(f"""Optimize this article for SEO.
TITLE: {topic_title}
FIRST 500 CHARS: {edited[:500]}
Respond with JSON: {{"seo_title": "...", "seo_description": "...", "keywords": ["..."]}}""",
model="qwen3.5:4b-mlx", temperature=0.3)
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.3)
# Agent 5: Real Fact Check (web-verified)
factcheck = real_fact_check(edited, topic_title)
@@ -930,7 +944,7 @@ ARTICLE:
{edited}
Return the expanded article in full Markdown. No JSON wrapper.""",
model="qwen3.5:4b-mlx", temperature=0.5, max_tokens=8192)
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.5, max_tokens=8192)
passed, issues = quality_gate(edited, topic_title, vertical)
if not passed:
@@ -1413,7 +1427,8 @@ def run_daily_pipeline(max_articles: int = 3):
log.warning(f"No CT IP for {vertical} — skipping publish")
continue
api_url = f"http://{ct_ip}:5000/api/publish"
port = vinfo.get("port", 80)
api_url = f"http://{ct_ip}:{port}/api/publish"
for article in articles:
try:
r = requests.post(api_url, json=article,
@@ -1421,6 +1436,11 @@ def run_daily_pipeline(max_articles: int = 3):
timeout=15)
if r.status_code in (200, 201):
log.info(f" 📤 Published to {vertical}: {article.get('title', '')[:60]}")
# Update article status in local DB
aid = article.get('topic_id')
if aid:
db.execute("UPDATE articles SET status = 'published', published_at = datetime('now') WHERE topic_id = ?", (aid,))
db.commit()
else:
log.warning(f"{vertical} API returned {r.status_code}: {r.text[:100]}")
except Exception as e:

View File

@@ -6,18 +6,47 @@ import os
import json
import sqlite3
import hashlib
import time
from pathlib import Path
from datetime import datetime, timedelta
from functools import wraps
from datetime import datetime
from flask import Flask, request, jsonify, render_template_string, g, abort, Response
try:
import markdown as _md
except ImportError:
_md = None
def md_to_html(text):
"""Convert Markdown to HTML for article rendering."""
if not text:
return ""
if _md is not None:
return _md.markdown(text, extensions=["extra", "sane_lists"])
# Minimal fallback (markdown lib not installed)
import re as _re
out = _re.sub(r"^#{1,6}\s+(.+)$", r"<h3>\1</h3>", text, flags=_re.M)
out = _re.sub(r"^\*\*(.+?)\*\*$", r"<strong>\1</strong>", out, flags=_re.M)
return "<p>" + out.replace("\n\n", "</p><p>").replace("\n", "<br>") + "</p>"
# ─── Config ────────────────────────────────────────────────────────
VERTICAL = os.environ.get("PUBLISHER_VERTICAL", "guides")
DOMAIN = f"{VERTICAL}.thetempleofdoom.com"
DB_PATH = Path(f"/var/lib/publisher/{VERTICAL}.db")
SECRET = os.environ.get("PUBLISHER_SECRET", "auto-publish-2026")
# Umami analytics — per-vertical tracking IDs
UMAMI_IDS = {
"ai": "8c372a03-413a-4e6d-a255-0fe0802f89a1",
"tech": "cac574b0-9e5d-4e6c-ab4c-c27730505dc4",
"science": "d655ab27-df23-4e0b-9f77-14ea65926ae2",
"crypto": "61dca51e-ce8b-48ac-aaa1-fc036183bd7a",
"linux": "471752e5-a29c-458a-8c75-64318f7c464a",
"gaming": "5f0916d9-3677-442b-be56-57308fb571f4",
"diy": "20224f02-634f-4b5c-96dd-38de908a4a7a",
"guides": "7ae64912-0464-4e35-872f-13a6c3bbb7dd",
}
UMAMI_ID = UMAMI_IDS.get(VERTICAL, "")
# Per-vertical identity
IDENTITIES = {
"ai": {
@@ -146,6 +175,7 @@ NETWORK_SITES = [
]
IDENTITY = IDENTITIES.get(VERTICAL, IDENTITIES["guides"])
IDENTITY = {**IDENTITY, "umami_id": UMAMI_ID}
app = Flask(__name__)
@@ -453,6 +483,17 @@ def sitemap():
return Response(build_sitemap_xml(), mimetype="application/xml")
@app.route("/robots.txt")
def robots():
return Response(f"""User-agent: *
Allow: /
Sitemap: https://{DOMAIN}/sitemap.xml
User-agent: GPTBot
Disallow: /
""", mimetype="text/plain")
@app.route("/tag/<tag>")
def tag_page(tag):
"""Aggregate all articles with a given tag."""
@@ -497,8 +538,10 @@ def api_publish():
slug = data.get("slug", "")
title = data.get("title", "")
content_html = data.get("content_html", data.get("content_md", ""))
content_md = data.get("content_md", "")
content_md = data.get("content_md") or data.get("content") or ""
content_html = data.get("content_html", "")
if not content_html and content_md:
content_html = md_to_html(content_md)
excerpt = data.get("excerpt", data.get("seo_description", ""))
seo_title = data.get("seo_title", title)
seo_description = data.get("seo_description", "")
@@ -794,6 +837,7 @@ HOME_TEMPLATE = """<!DOCTYPE html>
.hero-stats{flex-wrap:wrap;gap:0.75rem}
}
</style>
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
</head>
<body>
<header>
@@ -1058,6 +1102,7 @@ ARTICLE_TEMPLATE = """<!DOCTYPE html>
.subscribe-form input{flex:1;padding:0.6rem 0.75rem;background:var(--bg);border:1px solid var(--border);border-radius:6px;color:var(--text);font-size:0.9rem}
.subscribe-form button{background:var(--gradient);color:white;border:none;padding:0.6rem 1.25rem;border-radius:6px;cursor:pointer;font-weight:600;font-size:0.9rem}
</style>
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
</head>
<body>
<header>
@@ -1088,7 +1133,7 @@ ARTICLE_TEMPLATE = """<!DOCTYPE html>
</div>
<div class="article-content">
{{ article.content_html|safe }}
{{ (article.content_md or article.content_html)|md|safe }}
</div>
<footer class="article-footer">
@@ -1257,6 +1302,7 @@ SEARCH_TEMPLATE = """<!DOCTYPE html>
.result p{color:var(--text-muted);font-size:0.88rem}
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
</style>
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
</head>
<body>
<header>
@@ -1311,6 +1357,7 @@ TAG_TEMPLATE = """<!DOCTYPE html>
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
a{color:var(--accent)}
</style>
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
</head>
<body>
<header><nav><a href="/" class="logo">{{ name }}</a></nav></header>
@@ -1344,6 +1391,7 @@ NOT_FOUND_TEMPLATE = """<!DOCTYPE html>
p{color:var(--text-muted);margin:1rem 0}
a{color:var(--primary)}
</style>
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
</head>
<body>
<div>
@@ -1364,6 +1412,11 @@ def from_json_filter(s):
return []
@app.template_filter("md")
def md_filter(s):
return md_to_html(s or "")
# ─── Main ──────────────────────────────────────────────────────────
if __name__ == "__main__":
import argparse