Feed discovery engine: HN, GitHub, Google News, Reddit. 7/8 sites updated with trending topics.

core/feeds.py

@@ -0,0 +1,142 @@
+"""
+Content Feed Discovery Engine - pulls trending topics from real sources.
+HN, Reddit, GitHub, Google News, RSS. No auth required.
+"""
+import json, time, requests, re
+from datetime import datetime
+from urllib.parse import quote
+
+# Vertical → keywords for feed matching
+VERTICALS = {
+    "ai": ["ai", "machine learning", "llm", "gpt", "neural", "transformer", "openai", "deepmind", "anthropic"],
+    "tech": ["tech", "startup", "saas", "cloud", "api", "software", "apple", "google", "microsoft", "aws"],
+    "science": ["science", "research", "nasa", "space", "physics", "biology", "chemistry", "quantum", "climate"],
+    "crypto": ["crypto", "bitcoin", "ethereum", "defi", "nft", "web3", "blockchain", "solana", "stablecoin"],
+    "linux": ["linux", "kernel", "ubuntu", "debian", "fedora", "arch", "bash", "systemd", "gnome", "kde"],
+    "gaming": ["game", "gaming", "steam", "playstation", "xbox", "nintendo", "esports", "unreal", "unity"],
+    "diy": ["diy", "maker", "3d print", "raspberry", "arduino", "woodwork", "electronics", "repair", "build"],
+    "guides": ["how to", "guide", "tutorial", "tips", "productivity", "learn", "setup", "configure"],
+}
+
+def fetch_hn_top():
+    """Hacker News top stories - returns list of {title, url, score}."""
+    try:
+        r = requests.get("https://hacker-news.firebaseio.com/v0/topstories.json", timeout=10)
+        ids = r.json()[:20]
+        stories = []
+        for sid in ids[:20]:
+            item = requests.get(f"https://hacker-news.firebaseio.com/v0/item/{sid}.json", timeout=5).json()
+            if item and item.get("title"):
+                stories.append({"title": item["title"], "url": item.get("url",""), "score": item.get("score",0), "source": "hackernews"})
+        return stories
+    except Exception as e:
+        print(f"  HN fetch failed: {e}")
+        return []
+
+def fetch_reddit_hot(subreddit="all", limit=15):
+    """Reddit hot posts via RSS."""
+    try:
+        headers = {"User-Agent": "Hermes/1.0"}
+        url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit={limit}"
+        r = requests.get(url, headers=headers, timeout=10)
+        posts = r.json().get("data", {}).get("children", [])
+        return [{"title": p["data"]["title"], "url": p["data"]["url"], "score": p["data"]["score"], "source": f"reddit/r/{subreddit}"} for p in posts]
+    except Exception as e:
+        print(f"  Reddit fetch failed: {e}")
+        return []
+
+def fetch_github_trending():
+    """GitHub trending repos."""
+    try:
+        r = requests.get("https://api.github.com/search/repositories?q=stars:>100+pushed:>2026-07-01&sort=stars&per_page=15", timeout=10)
+        repos = r.json().get("items", [])
+        return [{"title": f"{repo['full_name']}: {repo.get('description','')}", "url": repo["html_url"], "score": repo["stargazers_count"], "source": "github"} for repo in repos]
+    except Exception as e:
+        print(f"  GitHub fetch failed: {e}")
+        return []
+
+def fetch_google_news(topic="technology"):
+    """Google News RSS."""
+    try:
+        url = f"https://news.google.com/rss/search?q={quote(topic)}&hl=en-US&gl=US&ceid=US:en"
+        r = requests.get(url, timeout=10)
+        titles = re.findall(r"<title>([^<]+)</title>", r.text)[2:12]  # Skip feed title and empty
+        return [{"title": t, "url": "", "score": 0, "source": "google news"} for t in titles if t and not t.startswith("See more")]
+    except Exception as e:
+        print(f"  Google News fetch failed: {e}")
+        return []
+
+def classify_vertical(title):
+    """Match a title to the best vertical."""
+    t = title.lower()
+    scores = {}
+    for vertical, keywords in VERTICALS.items():
+        score = sum(1 for kw in keywords if kw.lower() in t)
+        if score > 0:
+            scores[vertical] = score
+    if not scores:
+        return "guides"  # default
+    return max(scores, key=scores.get)
+
+def discover_all():
+    """Run all feed sources and return scored topics by vertical."""
+    all_items = []
+    
+    print("  Fetching HN...")
+    all_items.extend(fetch_hn_top())
+    
+    for sub in ["technology", "science", "programming", "gaming", "cryptocurrency", "diy", "linux"]:
+        print(f"  Fetching Reddit r/{sub}...")
+        all_items.extend(fetch_reddit_hot(sub, 10))
+    
+    print("  Fetching GitHub...")
+    all_items.extend(fetch_github_trending())
+    
+    for topic in ["technology", "science", "AI", "crypto", "Linux", "gaming"]:
+        print(f"  Fetching Google News: {topic}...")
+        all_items.extend(fetch_google_news(topic))
+    
+    # Dedupe and score
+    seen = set()
+    unique = []
+    for item in all_items:
+        key = item["title"][:80].lower()
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+    
+    # Classify
+    for item in unique:
+        item["vertical"] = classify_vertical(item["title"])
+    
+    # Group by vertical
+    by_vertical = {}
+    for item in unique:
+        v = item["vertical"]
+        by_vertical.setdefault(v, []).append(item)
+    
+    # Sort each vertical by score
+    for v in by_vertical:
+        by_vertical[v].sort(key=lambda x: x.get("score", 0), reverse=True)
+        by_vertical[v] = by_vertical[v][:5]  # Top 5 per vertical
+    
+    return by_vertical
+
+if __name__ == "__main__":
+    import sys
+    vertical = sys.argv[1] if len(sys.argv) > 1 else None
+    
+    print(f"Feed Discovery Engine - {datetime.now().isoformat()[:19]}")
+    results = discover_all()
+    
+    if vertical:
+        items = results.get(vertical, [])
+        print(f"\n{vertical.upper()} ({len(items)} topics):")
+        for i, item in enumerate(items):
+            print(f"  {i+1}. [{item['source']}] {item['title'][:80]} (score={item['score']})")
+    else:
+        total = sum(len(v) for v in results.values())
+        print(f"\n{total} topics across {len(results)} verticals:")
+        for v, items in sorted(results.items()):
+            print(f"  {v}: {len(items)} topics")
+