auto-publisher/core/feeds.py

"""
Content Feed Discovery Engine - pulls trending topics from real sources.
HN, Reddit, GitHub, Google News, RSS. No auth required.
"""
import json, time, requests, re
from datetime import datetime
from urllib.parse import quote

# Vertical → keywords for feed matching
VERTICALS = {
    "ai": ["ai", "machine learning", "llm", "gpt", "neural", "transformer", "openai", "deepmind", "anthropic"],
    "tech": ["tech", "startup", "saas", "cloud", "api", "software", "apple", "google", "microsoft", "aws"],
    "science": ["science", "research", "nasa", "space", "physics", "biology", "chemistry", "quantum", "climate"],
    "crypto": ["crypto", "bitcoin", "ethereum", "defi", "nft", "web3", "blockchain", "solana", "stablecoin"],
    "linux": ["linux", "kernel", "ubuntu", "debian", "fedora", "arch", "bash", "systemd", "gnome", "kde"],
    "gaming": ["game", "gaming", "steam", "playstation", "xbox", "nintendo", "esports", "unreal", "unity"],
    "diy": ["diy", "maker", "3d print", "raspberry", "arduino", "woodwork", "electronics", "repair", "build"],
    "guides": ["how to", "guide", "tutorial", "tips", "productivity", "learn", "setup", "configure"],
}

def fetch_hn_top():
    """Hacker News top stories - returns list of {title, url, score}."""
    try:
        r = requests.get("https://hacker-news.firebaseio.com/v0/topstories.json", timeout=10)
        ids = r.json()[:20]
        stories = []
        for sid in ids[:20]:
            item = requests.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", timeout=5).json()
            if item and item.get("title"):
                stories.append({"title": item["title"], "url": item.get("url",""), "score": item.get("score",0), "source": "hackernews"})
        return stories
    except Exception as e:
        print(f"  HN fetch failed: {e}")
        return []

def fetch_reddit_hot(subreddit="all", limit=15):
    """Reddit hot posts via RSS."""
    try:
        headers = {"User-Agent": "Hermes/1.0"}
        url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit={limit}"
        r = requests.get(url, headers=headers, timeout=10)
        posts = r.json().get("data", {}).get("children", [])
        return [{"title": p["data"]["title"], "url": p["data"]["url"], "score": p["data"]["score"], "source": f"reddit/r/{subreddit}"} for p in posts]
    except Exception as e:
        print(f"  Reddit fetch failed: {e}")
        return []

def fetch_github_trending():
    """GitHub trending repos."""
    try:
        r = requests.get("https://api.github.com/search/repositories?q=stars:>100+pushed:>2026-07-01&sort=stars&per_page=15", timeout=10)
        repos = r.json().get("items", [])
        return [{"title": f"{repo['full_name']}: {repo.get('description','')}", "url": repo["html_url"], "score": repo["stargazers_count"], "source": "github"} for repo in repos]
    except Exception as e:
        print(f"  GitHub fetch failed: {e}")
        return []

def fetch_google_news(topic="technology"):
    """Google News RSS."""
    try:
        url = f"https://news.google.com/rss/search?q={quote(topic)}&hl=en-US&gl=US&ceid=US:en"
        r = requests.get(url, timeout=10)
        titles = re.findall(r"<title>([^<]+)</title>", r.text)[2:12]  # Skip feed title and empty
        return [{"title": t, "url": "", "score": 0, "source": "google news"} for t in titles if t and not t.startswith("See more")]
    except Exception as e:
        print(f"  Google News fetch failed: {e}")
        return []

def classify_vertical(title):
    """Match a title to the best vertical."""
    t = title.lower()
    scores = {}
    for vertical, keywords in VERTICALS.items():
        score = sum(1 for kw in keywords if kw.lower() in t)
        if score > 0:
            scores[vertical] = score
    if not scores:
        return "guides"  # default
    return max(scores, key=scores.get)

def discover_all():
    """Run all feed sources and return scored topics by vertical."""
    all_items = []
    
    print("  Fetching HN...")
    all_items.extend(fetch_hn_top())
    
    for sub in ["technology", "science", "programming", "gaming", "cryptocurrency", "diy", "linux"]:
        print(f"  Fetching Reddit r/{sub}...")
        all_items.extend(fetch_reddit_hot(sub, 10))
    
    print("  Fetching GitHub...")
    all_items.extend(fetch_github_trending())
    
    for topic in ["technology", "science", "AI", "crypto", "Linux", "gaming"]:
        print(f"  Fetching Google News: {topic}...")
        all_items.extend(fetch_google_news(topic))
    
    # Dedupe and score
    seen = set()
    unique = []
    for item in all_items:
        key = item["title"][:80].lower()
        if key not in seen:
            seen.add(key)
            unique.append(item)
    
    # Classify
    for item in unique:
        item["vertical"] = classify_vertical(item["title"])
    
    # Group by vertical
    by_vertical = {}
    for item in unique:
        v = item["vertical"]
        by_vertical.setdefault(v, []).append(item)
    
    # Sort each vertical by score
    for v in by_vertical:
        by_vertical[v].sort(key=lambda x: x.get("score", 0), reverse=True)
        by_vertical[v] = by_vertical[v][:5]  # Top 5 per vertical
    
    return by_vertical

if __name__ == "__main__":
    import sys
    vertical = sys.argv[1] if len(sys.argv) > 1 else None
    
    print(f"Feed Discovery Engine - {datetime.now().isoformat()[:19]}")
    results = discover_all()
    
    if vertical:
        items = results.get(vertical, [])
        print(f"\n{vertical.upper()} ({len(items)} topics):")
        for i, item in enumerate(items):
            print(f"  {i+1}. [{item['source']}] {item['title'][:80]} (score={item['score']})")
    else:
        total = sum(len(v) for v in results.values())
        print(f"\n{total} topics across {len(results)} verticals:")
        for v, items in sorted(results.items()):
            print(f"  {v}: {len(items)} topics")