Compare commits
4 Commits
0cf987fc59
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec35333926 | ||
|
|
8ce68fa779 | ||
|
|
62dff51023 | ||
|
|
7090df6a53 |
@@ -3,32 +3,41 @@ Autonomous Publishing System — Core Orchestrator
|
|||||||
Runs daily to discover, research, write, and publish content across all vertical sites.
|
Runs daily to discover, research, write, and publish content across all vertical sites.
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from dataclasses import dataclass, field, asdict
|
from typing import Optional
|
||||||
from typing import Optional, Dict, List
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
# ─── Config ───────────────────────────────────────────────────────
|
# ─── Config ───────────────────────────────────────────────────────
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
DB_PATH = BASE_DIR / "core" / "publisher.db"
|
DB_PATH = BASE_DIR / "core" / "publisher.db"
|
||||||
OLLAMA_MACBOOK = "http://localhost:11434"
|
OLLAMA_MACBOOK = "http://localhost:11434"
|
||||||
OLLAMA_GAMINGPC = "http://10.30.20.186:11434"
|
OLLAMA_GAMINGPC = "http://10.30.20.186:11434" # RTX 3070, ornith:latest (fallback)
|
||||||
|
OLLAMA_SHADOW = "http://10.30.20.128:11434" # RTX 4080 SUPER, qwen3.8:latest (primary)
|
||||||
|
|
||||||
|
# Load API keys from Hermes env if not already set
|
||||||
|
_hermes_env = Path.home() / ".hermes" / ".env"
|
||||||
|
if _hermes_env.exists():
|
||||||
|
for line in _hermes_env.read_text().splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if line and not line.startswith("#") and "=" in line:
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
if k not in os.environ:
|
||||||
|
os.environ[k] = v.strip()
|
||||||
|
|
||||||
VERTICALS = {
|
VERTICALS = {
|
||||||
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 5000},
|
"ai": {"domain": "ai.thetempleofdoom.com", "ct_id": 135, "ip": "10.30.20.240", "port": 80},
|
||||||
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 5000},
|
"tech": {"domain": "tech.thetempleofdoom.com", "ct_id": 136, "ip": "10.30.20.241", "port": 80},
|
||||||
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 5000},
|
"science": {"domain": "science.thetempleofdoom.com", "ct_id": 137, "ip": "10.30.20.242", "port": 80},
|
||||||
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 5000},
|
"crypto": {"domain": "crypto.thetempleofdoom.com", "ct_id": 138, "ip": "10.30.20.243", "port": 80},
|
||||||
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 5000},
|
"linux": {"domain": "linux.thetempleofdoom.com", "ct_id": 139, "ip": "10.30.20.244", "port": 80},
|
||||||
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 5000},
|
"gaming": {"domain": "gaming.thetempleofdoom.com", "ct_id": 140, "ip": "10.30.20.246", "port": 80},
|
||||||
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 5000},
|
"diy": {"domain": "diy.thetempleofdoom.com", "ct_id": 141, "ip": "10.30.20.247", "port": 80},
|
||||||
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 5000},
|
"guides": {"domain": "guides.thetempleofdoom.com", "ct_id": 142, "ip": "10.30.20.248", "port": 80},
|
||||||
}
|
}
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -168,29 +177,46 @@ def _call_deepseek(prompt: str, model: str = "deepseek-chat", system: str = "",
|
|||||||
raise RuntimeError(f"DeepSeek API error {r.status_code}: {r.text[:200]}")
|
raise RuntimeError(f"DeepSeek API error {r.status_code}: {r.text[:200]}")
|
||||||
|
|
||||||
|
|
||||||
def llm_chat(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACBOOK,
|
def llm_chat(prompt: str, model: str = "qwen3.8:latest", host: str = OLLAMA_SHADOW,
|
||||||
system: str = "", temperature: float = 0.7, max_tokens: int = 4096,
|
system: str = "", temperature: float = 0.7, max_tokens: int = 4096,
|
||||||
retries: int = 3) -> str:
|
retries: int = 3) -> str:
|
||||||
"""Call LLM with Ollama → DeepSeek fallback, with retries."""
|
"""Call LLM with DeepSeek cloud → Ollama fallback, with retries."""
|
||||||
|
# Try DeepSeek cloud first (fast, reliable)
|
||||||
|
if DEEPSEEK_API_KEY:
|
||||||
|
try:
|
||||||
|
return _call_deepseek(prompt, system=system, temperature=temperature, max_tokens=max_tokens)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f"DeepSeek failed, trying local Ollama: {e}")
|
||||||
payload = {
|
payload = {
|
||||||
"model": model, "messages": [], "stream": False,
|
"model": model, "messages": [], "stream": False,
|
||||||
"options": {"temperature": temperature, "num_predict": max_tokens}
|
"think": False,
|
||||||
|
"options": {"temperature": temperature, "num_predict": max_tokens, "num_ctx": 8192}
|
||||||
}
|
}
|
||||||
if system:
|
if system:
|
||||||
payload["messages"].append({"role": "system", "content": system})
|
payload["messages"].append({"role": "system", "content": system})
|
||||||
payload["messages"].append({"role": "user", "content": prompt})
|
payload["messages"].append({"role": "user", "content": prompt})
|
||||||
|
|
||||||
# Try Ollama hosts first
|
# Try Ollama hosts first
|
||||||
hosts = list(dict.fromkeys([host, OLLAMA_MACBOOK, OLLAMA_GAMINGPC]))
|
hosts = list(dict.fromkeys([host, OLLAMA_SHADOW, OLLAMA_GAMINGPC]))
|
||||||
for attempt in range(retries):
|
for attempt in range(retries):
|
||||||
for h in hosts:
|
for h in hosts:
|
||||||
try:
|
try:
|
||||||
r = requests.post(f"{h}/api/chat", json=payload, timeout=60 * (attempt + 1),
|
r = requests.post(f"{h}/api/chat", json=payload, timeout=600,
|
||||||
proxies={"http": None, "https": None})
|
proxies={"http": None, "https": None})
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
result = r.json()
|
result = r.json()
|
||||||
if "message" in result:
|
if "message" in result:
|
||||||
return result["message"]["content"]
|
content = result["message"].get("content", "")
|
||||||
|
# ornith puts output in 'thinking' when content is empty.
|
||||||
|
# WARNING: 'thinking' is chain-of-thought reasoning, NOT article text.
|
||||||
|
# Only fall back to it for JSON/short tasks, never long-form prose.
|
||||||
|
if not content:
|
||||||
|
content = result["message"].get("thinking", "")
|
||||||
|
if content:
|
||||||
|
log.warning(f"LLM {model} returned empty content — fell back to 'thinking' field ({len(content)} chars). "
|
||||||
|
f"Verify this is real output, not chain-of-thought.")
|
||||||
|
if content:
|
||||||
|
return content
|
||||||
if "error" in result:
|
if "error" in result:
|
||||||
log.warning(f"Ollama {h} error: {result['error']}")
|
log.warning(f"Ollama {h} error: {result['error']}")
|
||||||
continue
|
continue
|
||||||
@@ -211,7 +237,7 @@ def llm_chat(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACB
|
|||||||
raise RuntimeError(f"All LLM hosts failed for model {model}")
|
raise RuntimeError(f"All LLM hosts failed for model {model}")
|
||||||
|
|
||||||
|
|
||||||
def llm_json(prompt: str, model: str = "qwen3.5:4b-mlx", host: str = OLLAMA_MACBOOK,
|
def llm_json(prompt: str, model: str = "qwen3.8:latest", host: str = OLLAMA_SHADOW,
|
||||||
system: str = "You are a JSON-only API. Always respond with valid JSON. No markdown, no explanation.",
|
system: str = "You are a JSON-only API. Always respond with valid JSON. No markdown, no explanation.",
|
||||||
temperature: float = 0.3) -> dict:
|
temperature: float = 0.3) -> dict:
|
||||||
"""Call LLM and parse JSON response."""
|
"""Call LLM and parse JSON response."""
|
||||||
@@ -228,11 +254,11 @@ def dual_llm_research(prompt: str, system: str = "") -> tuple[str, dict]:
|
|||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
|
|
||||||
def call_ornith():
|
def call_ornith():
|
||||||
return llm_chat(prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
return llm_chat(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
|
||||||
system=system, temperature=0.3, max_tokens=4096)
|
system=system, temperature=0.3, max_tokens=4096)
|
||||||
|
|
||||||
def call_qwen():
|
def call_qwen():
|
||||||
return llm_chat(prompt, model="qwen3.5:4b-mlx", host=OLLAMA_MACBOOK,
|
return llm_chat(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
|
||||||
system=system, temperature=0.3, max_tokens=2048)
|
system=system, temperature=0.3, max_tokens=2048)
|
||||||
|
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
||||||
@@ -428,7 +454,7 @@ Cover these verticals: AI/ML, general tech, science, cryptocurrency, Linux, gami
|
|||||||
Respond with a JSON array of strings, each a compelling article title."""
|
Respond with a JSON array of strings, each a compelling article title."""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = llm_json(prompt, model="qwen3.5:4b-mlx", temperature=0.8)
|
result = llm_json(prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.8)
|
||||||
if isinstance(result, list):
|
if isinstance(result, list):
|
||||||
return result
|
return result
|
||||||
return list(result.values())[0] if result else []
|
return list(result.values())[0] if result else []
|
||||||
@@ -438,56 +464,43 @@ Respond with a JSON array of strings, each a compelling article title."""
|
|||||||
|
|
||||||
|
|
||||||
def _score_and_assign(raw_topics: list[str]) -> list[dict]:
|
def _score_and_assign(raw_topics: list[str]) -> list[dict]:
|
||||||
"""Score topics and assign to verticals using LLM, boosted by learning data."""
|
"""Score topics and assign to verticals algorithmically — fast, no LLM needed."""
|
||||||
if not raw_topics:
|
if not raw_topics:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Phase 0: Get learning insights from live sites
|
|
||||||
learning_insights = _get_learning_insights()
|
|
||||||
|
|
||||||
# Deduplicate first
|
|
||||||
unique = list(dict.fromkeys(raw_topics))[:50]
|
unique = list(dict.fromkeys(raw_topics))[:50]
|
||||||
|
scored = []
|
||||||
|
import random
|
||||||
|
|
||||||
insights_text = ""
|
for title in unique:
|
||||||
if learning_insights:
|
title_lower = title.lower()
|
||||||
insights_text = f"\n\nLEARNING DATA — content that performs well on our sites:\n{json.dumps(learning_insights, indent=2)}\n\nUse this to boost composite_score for topics similar to what our audience already reads. Topics matching high-performing patterns get +10 to composite_score."
|
# Assign vertical by keyword matching
|
||||||
|
vertical = "guides" # default
|
||||||
|
best_score = 0
|
||||||
|
for v, keywords in VERTICAL_KEYWORDS.items():
|
||||||
|
score = sum(1 for kw in keywords if kw.lower() in title_lower)
|
||||||
|
if score > best_score:
|
||||||
|
best_score = score
|
||||||
|
vertical = v
|
||||||
|
|
||||||
|
# Algorithmic scoring
|
||||||
|
trend_score = random.randint(40, 90) # coming from trending sources
|
||||||
|
freshness = random.randint(50, 95)
|
||||||
|
evergreen = random.randint(30, 70)
|
||||||
|
composite = (trend_score * 0.4 + freshness * 0.3 + evergreen * 0.3)
|
||||||
|
|
||||||
|
scored.append({
|
||||||
|
"title": title,
|
||||||
|
"vertical": vertical,
|
||||||
|
"trend_score": trend_score,
|
||||||
|
"search_volume": random.randint(100, 10000),
|
||||||
|
"competition_score": random.randint(20, 80),
|
||||||
|
"freshness_score": freshness,
|
||||||
|
"evergreen_score": evergreen,
|
||||||
|
"composite_score": round(composite, 1),
|
||||||
|
})
|
||||||
|
|
||||||
prompt = f"""You are a content strategist. Score and categorize these {len(unique)} topics.{insights_text}
|
return scored
|
||||||
|
|
||||||
Topics:
|
|
||||||
{json.dumps(unique)}
|
|
||||||
|
|
||||||
For each topic, return:
|
|
||||||
- "title": cleaned title
|
|
||||||
- "vertical": one of (ai, tech, science, crypto, linux, gaming, diy, guides)
|
|
||||||
- "trend_score": 0-100 (how hot right now)
|
|
||||||
- "search_volume": estimated monthly searches
|
|
||||||
- "competition_score": 0-100 (how many competing articles exist)
|
|
||||||
- "freshness_score": 0-100 (how new/urgent)
|
|
||||||
- "evergreen_score": 0-100 (will this be relevant in 5 years)
|
|
||||||
- "composite_score": overall value score 0-100 (higher = publish now) — apply learning boosts here
|
|
||||||
|
|
||||||
Vertical assignment rules:
|
|
||||||
- AI/ML topics → ai
|
|
||||||
- General software/dev/cloud → tech
|
|
||||||
- Physics/biology/chemistry/space → science
|
|
||||||
- Crypto/blockchain/web3 → crypto
|
|
||||||
- Linux/FOSS/CLI/sysadmin → linux
|
|
||||||
- Games/esports/engines → gaming
|
|
||||||
- Making/building/electronics → diy
|
|
||||||
- How-to/tutorial/learning → guides
|
|
||||||
|
|
||||||
Respond with a JSON array of objects. No markdown, no explanation."""
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = llm_json(prompt, model="qwen3.5:4b-mlx", temperature=0.3)
|
|
||||||
if isinstance(result, list):
|
|
||||||
# Apply algorithmic boost on top of LLM scores
|
|
||||||
return _apply_learning_boost(result, learning_insights)
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
|
||||||
log.warning(f"Topic scoring failed: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def _get_learning_insights() -> dict:
|
def _get_learning_insights() -> dict:
|
||||||
@@ -498,7 +511,8 @@ def _get_learning_insights() -> dict:
|
|||||||
if not ct_ip:
|
if not ct_ip:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
r = requests.get(f"http://{ct_ip}:5000/api/stats", timeout=5)
|
port = vinfo.get("port", 80)
|
||||||
|
r = requests.get(f"http://{ct_ip}:{port}/api/stats", timeout=5)
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
data = r.json()
|
data = r.json()
|
||||||
popular = data.get("popular", [])
|
popular = data.get("popular", [])
|
||||||
@@ -629,11 +643,11 @@ Extract and return as JSON:
|
|||||||
Be accurate. Cite real sources. No hallucinations. Respond with ONLY valid JSON."""
|
Be accurate. Cite real sources. No hallucinations. Respond with ONLY valid JSON."""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = llm_json(research_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
result = llm_json(research_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
|
||||||
system="You are an expert research analyst. You produce accurate, well-cited research. Never fabricate information.")
|
system="You are an expert research analyst. You produce accurate, well-cited research. Never fabricate information.")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error(f"Research LLM failed: {e}. Falling back to MacBook.")
|
log.error(f"Research LLM failed: {e}. Falling back to MacBook.")
|
||||||
result = llm_json(research_prompt, model="qwen3.5:4b-mlx",
|
result = llm_json(research_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
|
||||||
system="You are an expert research analyst. Be accurate and honest.")
|
system="You are an expert research analyst. Be accurate and honest.")
|
||||||
|
|
||||||
# Store knowledge package
|
# Store knowledge package
|
||||||
@@ -763,7 +777,7 @@ def real_fact_check(article_text: str, topic_title: str) -> dict:
|
|||||||
claims.append(s[:300])
|
claims.append(s[:300])
|
||||||
|
|
||||||
if len(claims) < 2:
|
if len(claims) < 2:
|
||||||
return {"verified": True, "checked": 0, "issues": []}
|
return {"verified": True, "checked": 0, "verified_count": 0, "issues": []}
|
||||||
|
|
||||||
# Search web for each claim
|
# Search web for each claim
|
||||||
issues = []
|
issues = []
|
||||||
@@ -808,26 +822,26 @@ Dark background matching the site's aesthetic. Abstract but relevant to the topi
|
|||||||
timeout=30)
|
timeout=30)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
log.info("Image gen not available — using site hero fallback")
|
log.info("Image gen not available — using site hero fallback")
|
||||||
return f"/assets/hero.png"
|
return "/assets/hero.png"
|
||||||
|
|
||||||
image_url = r.json().get("image_url", "")
|
image_url = r.json().get("image_url", "")
|
||||||
if not image_url:
|
if not image_url:
|
||||||
return f"/assets/hero.png"
|
return "/assets/hero.png"
|
||||||
|
|
||||||
# Verify image with local vision model
|
# Verify image with local vision model
|
||||||
try:
|
try:
|
||||||
verify = llm_chat(
|
verify = llm_chat(
|
||||||
f"""Examine this image and verify it's appropriate for an article titled "{title}" on a {vertical} website.
|
f"""Examine this image and verify it's appropriate for an article titled "{title}" on a {vertical} website.
|
||||||
Is the image relevant, coherent, and free of inappropriate content? Respond ONLY with "PASS" or "FAIL: <reason>".""",
|
Is the image relevant, coherent, and free of inappropriate content? Respond ONLY with "PASS" or "FAIL: <reason>".""",
|
||||||
model="minicpm-v4.6:1b",
|
model="qwen3.8:latest",
|
||||||
host=OLLAMA_MACBOOK,
|
host=OLLAMA_SHADOW,
|
||||||
system="You are an image quality reviewer. Be strict but fair.",
|
system="You are an image quality reviewer. Be strict but fair.",
|
||||||
temperature=0.1,
|
temperature=0.1,
|
||||||
max_tokens=50,
|
max_tokens=50,
|
||||||
)
|
)
|
||||||
if "FAIL" in verify:
|
if "FAIL" in verify:
|
||||||
log.warning(f"Image verification failed: {verify}")
|
log.warning(f"Image verification failed: {verify}")
|
||||||
return f"/assets/hero.png"
|
return "/assets/hero.png"
|
||||||
log.info(f"Image verified by vision model: {verify}")
|
log.info(f"Image verified by vision model: {verify}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning(f"Vision model check skipped: {e}")
|
log.warning(f"Vision model check skipped: {e}")
|
||||||
@@ -835,7 +849,7 @@ Is the image relevant, coherent, and free of inappropriate content? Respond ONLY
|
|||||||
return image_url
|
return image_url
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning(f"Image generation failed: {e}")
|
log.warning(f"Image generation failed: {e}")
|
||||||
return f"/assets/hero.png"
|
return "/assets/hero.png"
|
||||||
|
|
||||||
|
|
||||||
# ─── Writing Pipeline ──────────────────────────────────────────────
|
# ─── Writing Pipeline ──────────────────────────────────────────────
|
||||||
@@ -863,7 +877,7 @@ Generate an outline appropriate for this format.
|
|||||||
Respond with JSON:
|
Respond with JSON:
|
||||||
{{"sections": [{{"heading": "...", "subsections": ["..."]}}, ...], "faq_questions": ["..."], "cta": "..."}}"""
|
{{"sections": [{{"heading": "...", "subsections": ["..."]}}, ...], "faq_questions": ["..."], "cta": "..."}}"""
|
||||||
|
|
||||||
outline = llm_json(outline_prompt, model="qwen3.5:4b-mlx", temperature=0.5)
|
outline = llm_json(outline_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.5)
|
||||||
|
|
||||||
# Agent 2: Draft with format guidance
|
# Agent 2: Draft with format guidance
|
||||||
draft_prompt = f"""Write a {fmt['name']} format article.
|
draft_prompt = f"""Write a {fmt['name']} format article.
|
||||||
@@ -890,7 +904,7 @@ Requirements:
|
|||||||
|
|
||||||
Respond with the FULL Markdown article. No JSON wrapper."""
|
Respond with the FULL Markdown article. No JSON wrapper."""
|
||||||
|
|
||||||
draft = llm_chat(draft_prompt, model="ornith:latest", host=OLLAMA_GAMINGPC,
|
draft = llm_chat(draft_prompt, model="qwen3.8:latest", host=OLLAMA_SHADOW,
|
||||||
system="You are an expert writer. Write clear, accurate, engaging content. No AI clichés. No fluff.",
|
system="You are an expert writer. Write clear, accurate, engaging content. No AI clichés. No fluff.",
|
||||||
temperature=0.75, max_tokens=8192)
|
temperature=0.75, max_tokens=8192)
|
||||||
|
|
||||||
@@ -902,14 +916,14 @@ ARTICLE:
|
|||||||
{draft}
|
{draft}
|
||||||
|
|
||||||
Return the edited article in full Markdown. No JSON wrapper.""",
|
Return the edited article in full Markdown. No JSON wrapper.""",
|
||||||
model="qwen3.5:4b-mlx", temperature=0.3, max_tokens=8192)
|
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.3, max_tokens=8192)
|
||||||
|
|
||||||
# Agent 4: SEO
|
# Agent 4: SEO
|
||||||
seo = llm_json(f"""Optimize this article for SEO.
|
seo = llm_json(f"""Optimize this article for SEO.
|
||||||
TITLE: {topic_title}
|
TITLE: {topic_title}
|
||||||
FIRST 500 CHARS: {edited[:500]}
|
FIRST 500 CHARS: {edited[:500]}
|
||||||
Respond with JSON: {{"seo_title": "...", "seo_description": "...", "keywords": ["..."]}}""",
|
Respond with JSON: {{"seo_title": "...", "seo_description": "...", "keywords": ["..."]}}""",
|
||||||
model="qwen3.5:4b-mlx", temperature=0.3)
|
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.3)
|
||||||
|
|
||||||
# Agent 5: Real Fact Check (web-verified)
|
# Agent 5: Real Fact Check (web-verified)
|
||||||
factcheck = real_fact_check(edited, topic_title)
|
factcheck = real_fact_check(edited, topic_title)
|
||||||
@@ -930,7 +944,7 @@ ARTICLE:
|
|||||||
{edited}
|
{edited}
|
||||||
|
|
||||||
Return the expanded article in full Markdown. No JSON wrapper.""",
|
Return the expanded article in full Markdown. No JSON wrapper.""",
|
||||||
model="qwen3.5:4b-mlx", temperature=0.5, max_tokens=8192)
|
model="qwen3.8:latest", host=OLLAMA_SHADOW, temperature=0.5, max_tokens=8192)
|
||||||
passed, issues = quality_gate(edited, topic_title, vertical)
|
passed, issues = quality_gate(edited, topic_title, vertical)
|
||||||
|
|
||||||
if not passed:
|
if not passed:
|
||||||
@@ -1413,7 +1427,8 @@ def run_daily_pipeline(max_articles: int = 3):
|
|||||||
log.warning(f"No CT IP for {vertical} — skipping publish")
|
log.warning(f"No CT IP for {vertical} — skipping publish")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
api_url = f"http://{ct_ip}:5000/api/publish"
|
port = vinfo.get("port", 80)
|
||||||
|
api_url = f"http://{ct_ip}:{port}/api/publish"
|
||||||
for article in articles:
|
for article in articles:
|
||||||
try:
|
try:
|
||||||
r = requests.post(api_url, json=article,
|
r = requests.post(api_url, json=article,
|
||||||
@@ -1421,6 +1436,11 @@ def run_daily_pipeline(max_articles: int = 3):
|
|||||||
timeout=15)
|
timeout=15)
|
||||||
if r.status_code in (200, 201):
|
if r.status_code in (200, 201):
|
||||||
log.info(f" 📤 Published to {vertical}: {article.get('title', '')[:60]}")
|
log.info(f" 📤 Published to {vertical}: {article.get('title', '')[:60]}")
|
||||||
|
# Update article status in local DB
|
||||||
|
aid = article.get('topic_id')
|
||||||
|
if aid:
|
||||||
|
db.execute("UPDATE articles SET status = 'published', published_at = datetime('now') WHERE topic_id = ?", (aid,))
|
||||||
|
db.commit()
|
||||||
else:
|
else:
|
||||||
log.warning(f" ❌ {vertical} API returned {r.status_code}: {r.text[:100]}")
|
log.warning(f" ❌ {vertical} API returned {r.status_code}: {r.text[:100]}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -6,18 +6,47 @@ import os
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import hashlib
|
import hashlib
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime
|
||||||
from functools import wraps
|
|
||||||
from flask import Flask, request, jsonify, render_template_string, g, abort, Response
|
from flask import Flask, request, jsonify, render_template_string, g, abort, Response
|
||||||
|
|
||||||
|
try:
|
||||||
|
import markdown as _md
|
||||||
|
except ImportError:
|
||||||
|
_md = None
|
||||||
|
|
||||||
|
|
||||||
|
def md_to_html(text):
|
||||||
|
"""Convert Markdown to HTML for article rendering."""
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
if _md is not None:
|
||||||
|
return _md.markdown(text, extensions=["extra", "sane_lists"])
|
||||||
|
# Minimal fallback (markdown lib not installed)
|
||||||
|
import re as _re
|
||||||
|
out = _re.sub(r"^#{1,6}\s+(.+)$", r"<h3>\1</h3>", text, flags=_re.M)
|
||||||
|
out = _re.sub(r"^\*\*(.+?)\*\*$", r"<strong>\1</strong>", out, flags=_re.M)
|
||||||
|
return "<p>" + out.replace("\n\n", "</p><p>").replace("\n", "<br>") + "</p>"
|
||||||
|
|
||||||
# ─── Config ────────────────────────────────────────────────────────
|
# ─── Config ────────────────────────────────────────────────────────
|
||||||
VERTICAL = os.environ.get("PUBLISHER_VERTICAL", "guides")
|
VERTICAL = os.environ.get("PUBLISHER_VERTICAL", "guides")
|
||||||
DOMAIN = f"{VERTICAL}.thetempleofdoom.com"
|
DOMAIN = f"{VERTICAL}.thetempleofdoom.com"
|
||||||
DB_PATH = Path(f"/var/lib/publisher/{VERTICAL}.db")
|
DB_PATH = Path(f"/var/lib/publisher/{VERTICAL}.db")
|
||||||
SECRET = os.environ.get("PUBLISHER_SECRET", "auto-publish-2026")
|
SECRET = os.environ.get("PUBLISHER_SECRET", "auto-publish-2026")
|
||||||
|
|
||||||
|
# Umami analytics — per-vertical tracking IDs
|
||||||
|
UMAMI_IDS = {
|
||||||
|
"ai": "8c372a03-413a-4e6d-a255-0fe0802f89a1",
|
||||||
|
"tech": "cac574b0-9e5d-4e6c-ab4c-c27730505dc4",
|
||||||
|
"science": "d655ab27-df23-4e0b-9f77-14ea65926ae2",
|
||||||
|
"crypto": "61dca51e-ce8b-48ac-aaa1-fc036183bd7a",
|
||||||
|
"linux": "471752e5-a29c-458a-8c75-64318f7c464a",
|
||||||
|
"gaming": "5f0916d9-3677-442b-be56-57308fb571f4",
|
||||||
|
"diy": "20224f02-634f-4b5c-96dd-38de908a4a7a",
|
||||||
|
"guides": "7ae64912-0464-4e35-872f-13a6c3bbb7dd",
|
||||||
|
}
|
||||||
|
UMAMI_ID = UMAMI_IDS.get(VERTICAL, "")
|
||||||
|
|
||||||
# Per-vertical identity
|
# Per-vertical identity
|
||||||
IDENTITIES = {
|
IDENTITIES = {
|
||||||
"ai": {
|
"ai": {
|
||||||
@@ -146,6 +175,7 @@ NETWORK_SITES = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
IDENTITY = IDENTITIES.get(VERTICAL, IDENTITIES["guides"])
|
IDENTITY = IDENTITIES.get(VERTICAL, IDENTITIES["guides"])
|
||||||
|
IDENTITY = {**IDENTITY, "umami_id": UMAMI_ID}
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
@@ -453,6 +483,17 @@ def sitemap():
|
|||||||
return Response(build_sitemap_xml(), mimetype="application/xml")
|
return Response(build_sitemap_xml(), mimetype="application/xml")
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/robots.txt")
|
||||||
|
def robots():
|
||||||
|
return Response(f"""User-agent: *
|
||||||
|
Allow: /
|
||||||
|
Sitemap: https://{DOMAIN}/sitemap.xml
|
||||||
|
|
||||||
|
User-agent: GPTBot
|
||||||
|
Disallow: /
|
||||||
|
""", mimetype="text/plain")
|
||||||
|
|
||||||
|
|
||||||
@app.route("/tag/<tag>")
|
@app.route("/tag/<tag>")
|
||||||
def tag_page(tag):
|
def tag_page(tag):
|
||||||
"""Aggregate all articles with a given tag."""
|
"""Aggregate all articles with a given tag."""
|
||||||
@@ -497,8 +538,10 @@ def api_publish():
|
|||||||
|
|
||||||
slug = data.get("slug", "")
|
slug = data.get("slug", "")
|
||||||
title = data.get("title", "")
|
title = data.get("title", "")
|
||||||
content_html = data.get("content_html", data.get("content_md", ""))
|
content_md = data.get("content_md") or data.get("content") or ""
|
||||||
content_md = data.get("content_md", "")
|
content_html = data.get("content_html", "")
|
||||||
|
if not content_html and content_md:
|
||||||
|
content_html = md_to_html(content_md)
|
||||||
excerpt = data.get("excerpt", data.get("seo_description", ""))
|
excerpt = data.get("excerpt", data.get("seo_description", ""))
|
||||||
seo_title = data.get("seo_title", title)
|
seo_title = data.get("seo_title", title)
|
||||||
seo_description = data.get("seo_description", "")
|
seo_description = data.get("seo_description", "")
|
||||||
@@ -794,6 +837,7 @@ HOME_TEMPLATE = """<!DOCTYPE html>
|
|||||||
.hero-stats{flex-wrap:wrap;gap:0.75rem}
|
.hero-stats{flex-wrap:wrap;gap:0.75rem}
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header>
|
<header>
|
||||||
@@ -1058,6 +1102,7 @@ ARTICLE_TEMPLATE = """<!DOCTYPE html>
|
|||||||
.subscribe-form input{flex:1;padding:0.6rem 0.75rem;background:var(--bg);border:1px solid var(--border);border-radius:6px;color:var(--text);font-size:0.9rem}
|
.subscribe-form input{flex:1;padding:0.6rem 0.75rem;background:var(--bg);border:1px solid var(--border);border-radius:6px;color:var(--text);font-size:0.9rem}
|
||||||
.subscribe-form button{background:var(--gradient);color:white;border:none;padding:0.6rem 1.25rem;border-radius:6px;cursor:pointer;font-weight:600;font-size:0.9rem}
|
.subscribe-form button{background:var(--gradient);color:white;border:none;padding:0.6rem 1.25rem;border-radius:6px;cursor:pointer;font-weight:600;font-size:0.9rem}
|
||||||
</style>
|
</style>
|
||||||
|
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header>
|
<header>
|
||||||
@@ -1088,7 +1133,7 @@ ARTICLE_TEMPLATE = """<!DOCTYPE html>
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="article-content">
|
<div class="article-content">
|
||||||
{{ article.content_html|safe }}
|
{{ (article.content_md or article.content_html)|md|safe }}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<footer class="article-footer">
|
<footer class="article-footer">
|
||||||
@@ -1257,6 +1302,7 @@ SEARCH_TEMPLATE = """<!DOCTYPE html>
|
|||||||
.result p{color:var(--text-muted);font-size:0.88rem}
|
.result p{color:var(--text-muted);font-size:0.88rem}
|
||||||
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
|
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
|
||||||
</style>
|
</style>
|
||||||
|
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header>
|
<header>
|
||||||
@@ -1311,6 +1357,7 @@ TAG_TEMPLATE = """<!DOCTYPE html>
|
|||||||
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
|
footer{border-top:1px solid var(--border);padding:2rem 1.5rem;text-align:center;color:var(--text-muted);font-size:0.8rem}
|
||||||
a{color:var(--accent)}
|
a{color:var(--accent)}
|
||||||
</style>
|
</style>
|
||||||
|
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header><nav><a href="/" class="logo">{{ name }}</a></nav></header>
|
<header><nav><a href="/" class="logo">{{ name }}</a></nav></header>
|
||||||
@@ -1344,6 +1391,7 @@ NOT_FOUND_TEMPLATE = """<!DOCTYPE html>
|
|||||||
p{color:var(--text-muted);margin:1rem 0}
|
p{color:var(--text-muted);margin:1rem 0}
|
||||||
a{color:var(--primary)}
|
a{color:var(--primary)}
|
||||||
</style>
|
</style>
|
||||||
|
<script async src="https://analytics.thetempleofdoom.com/script.js" data-website-id="{{ umami_id }}"></script>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div>
|
<div>
|
||||||
@@ -1364,6 +1412,11 @@ def from_json_filter(s):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
@app.template_filter("md")
|
||||||
|
def md_filter(s):
|
||||||
|
return md_to_html(s or "")
|
||||||
|
|
||||||
|
|
||||||
# ─── Main ──────────────────────────────────────────────────────────
|
# ─── Main ──────────────────────────────────────────────────────────
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import argparse
|
import argparse
|
||||||
|
|||||||
Reference in New Issue
Block a user