v2: quick-rule fast path + content signals (deterministic first), parallel Nimble w/ warmup+keep_alive, live pulse stats, apply toast, self-healing db

app.py

@@ -77,12 +77,61 @@ CREATE TABLE IF NOT EXISTS sessions(
 """)
     c.commit(); c.close()
 
+# deterministic fast-path: obvious extensions never need the LLM (instant + accurate)
+QUICK_RULES = {
+ ".pdf": ("document", None, None, 0.75), ".doc": ("document", None, None, 0.75), ".docx": ("document", None, None, 0.75),
+ ".xlsx": ("document", "finance", None, 0.8), ".xls": ("document", "finance", None, 0.8), ".pptx": ("document", "work", None, 0.8),
+ ".txt": ("document", None, None, 0.6), ".md": ("document", None, None, 0.6), ".csv": ("dataset", None, None, 0.8),
+ ".jpg": ("image", "photography", None, 0.9), ".jpeg": ("image", "photography", None, 0.9), ".png": ("image", None, None, 0.85),
+ ".heic": ("image", "photography", None, 0.9), ".gif": ("image", None, None, 0.85), ".webp": ("image", None, None, 0.85),
+ ".psd": ("image", None, None, 0.8),
+ ".mp4": ("video", None, None, 0.9), ".mov": ("video", None, None, 0.9), ".mkv": ("video", None, None, 0.9), ".avi": ("video", None, None, 0.9),
+ ".mp3": ("audio", None, None, 0.9), ".wav": ("audio", None, None, 0.9), ".flac": ("audio", None, None, 0.9),
+ ".zip": ("archive", None, None, 0.85), ".rar": ("archive", None, None, 0.85), ".7z": ("archive", None, None, 0.85),
+ ".tar": ("archive", None, None, 0.85), ".gz": ("archive", None, None, 0.85), ".iso": ("archive", None, None, 0.8),
+ ".py": ("source_code", "programming", None, 0.92), ".js": ("source_code", "programming", None, 0.92), ".ts": ("source_code", "programming", None, 0.92),
+ ".sh": ("source_code", "linux", None, 0.9), ".go": ("source_code", "programming", None, 0.92), ".rs": ("source_code", "programming", None, 0.92),
+ ".c": ("source_code", "programming", None, 0.9), ".cpp": ("source_code", "programming", None, 0.9), ".java": ("source_code", "programming", None, 0.92),
+ ".exe": ("software", None, None, 0.9), ".msi": ("software", None, None, 0.9), ".dmg": ("software", None, None, 0.9), ".app": ("software", None, None, 0.9),
+ ".deb": ("software", "linux", None, 0.9), ".rpm": ("software", "linux", None, 0.9), ".apk": ("software", None, None, 0.9),
+ ".ova": ("dataset", None, None, 0.7), ".vmdk": ("dataset", None, None, 0.7), ".sqlite": ("dataset", None, None, 0.8), ".db": ("dataset", None, None, 0.8),
+ ".epub": ("ebook", None, None, 0.95), ".mobi": ("ebook", None, None, 0.95), ".azw3": ("ebook", None, None, 0.95), ".djvu": ("ebook", None, None, 0.9),
+}
+# exts that LOOK like ebooks/docs but need content sniffing (pdf can be either) — handled by classifier
+
+# content signals — deterministic, beat quick rules when the content screams (receipt, fullz, hacking…)
+SIGNALS = [
+ (r"receipt|invoice|order confirmation|statement|payment|charged \$|billing", "financial", "finance", "important", "organize", 0.97),
+ (r"cvv|fullz|ssn|card number|carding|dump[s]? ", "personal", "unknown", "critical", "quarantine", 0.95),
+ (r"hacking|exploit|keylogger|rat |botnet|malware", "document", "cybersecurity", "important", "quarantine", 0.93),
+ (r"chapter one|novel|fiction|once upon", "document", "personal", "important", "organize", 0.9),
+]
+
+def sniff_content(p, ext, size):
+    if ext not in (".txt", ".md", ".csv", ".json", ".log", ".pdf", ".html", ".cfg", ".ini") or size > 5_000_000:
+        return None
+    try:
+        if ext == ".pdf":
+            from pypdf import PdfReader
+            txt = " ".join((pg.extract_text() or "") for pg in PdfReader(p).pages[:2])
+        else:
+            txt = open(p, "r", errors="replace").read(4000)
+    except Exception:
+        return None
+    low = txt.lower()
+    for rx, fc, sub, imp, act, conf in SIGNALS:
+        if re.search(rx, low):
+            return {"file_class": fc, "subject": sub, "importance": imp, "action": act,
+                    "confidence": conf, "rename": None, "preview": re.sub(r"\s+", " ", txt)[:600]}
+    return None
+
 # ---------- SCANNER ----------
 SKIP_DIRS = {".Trashes", ".Spotlight-V100", ".TemporaryItems", ".fseventsd", "System Volume Information",
              "$RECYCLE.BIN", "node_modules", "__pycache__", ".git", "Library", ".Trash"}
 JUNK = (".DS_Store",)
 
 def scan(root, max_files=5000, min_size=0, max_size=0):
+    init_db()
     root = os.path.abspath(os.path.expanduser(root))
     c = db()
     n = 0
@@ -99,6 +148,22 @@ def scan(root, max_files=5000, min_size=0, max_size=0):
             if st.st_size < min_size or (max_size and st.st_size > max_size):
                 continue
             try:
+                qr = QUICK_RULES.get(os.path.splitext(f)[1].lower())
+                if qr:
+                    fc, sub, imp, conf = qr
+                    sig = sniff_content(p, os.path.splitext(f)[1].lower(), st.st_size)
+                    if sig:
+                        fc, sub, imp, conf = sig["file_class"], sig["subject"], sig["importance"], sig["confidence"]
+                        c.execute("""INSERT OR IGNORE INTO files(path,name,ext,size,mtime,file_class,subject,importance,confidence,action,preview,status)
+                                     VALUES(?,?,?,?,?,?,?,?,?,?,?, 'classified')""",
+                                  (p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime, fc, sub, imp, conf, sig["action"], sig["preview"]))
+                        n += 1
+                        continue
+                    c.execute("""INSERT OR IGNORE INTO files(path,name,ext,size,mtime,file_class,subject,importance,confidence,action,status)
+                                 VALUES(?,?,?,?,?,?,?,?,?,'organize','classified')""",
+                              (p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime, fc, sub, imp or 'normal', conf))
+                    n += 1
+                    continue
                 c.execute("INSERT OR IGNORE INTO files(path,name,ext,size,mtime,status) VALUES(?,?,?,?,?, 'scanned')",
                           (p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime))
             except sqlite3.Error:
@@ -124,15 +189,18 @@ def scan_thread(root, max_files, min_size, max_size):
         refresh_counts()
 
 def refresh_counts():
-    c = db()
+    try:
-    for k, q in [("files", "SELECT COUNT(*) FROM files"),
+        c = db()
+        for k, q in [("files", "SELECT COUNT(*) FROM files"),
                  ("scanned", "SELECT COUNT(*) FROM files WHERE file_class IS NULL"),
                  ("classified", "SELECT COUNT(*) FROM files WHERE file_class IS NOT NULL"),
                  ("dupes", "SELECT COUNT(*) FROM files WHERE sha256 IS NOT NULL AND sha256 IN (SELECT sha256 FROM files WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(*)>1)"),
                  ("pending", "SELECT COUNT(*) FROM files WHERE status='approved'"),
                  ("moved", "SELECT COUNT(*) FROM files WHERE status='moved'")]:
-        STATE[k] = c.execute(q).fetchone()[0]
+            STATE[k] = c.execute(q).fetchone()[0]
-    c.close()
+        c.close()
+    except Exception:
+        pass
 
 # ---------- EXTRACTOR ----------
 def extract_preview(p, ext, size):
@@ -198,7 +266,16 @@ ENUMS = {
  "action": ["keep","organize","archive","quarantine","review"],
 }
 def classify_one(row):
-    prompt = ("Classify this file. Reply ONLY minified JSON with keys file_class, subject, importance, action, confidence, rename. No other text.\n"
+    prompt = ("You are a file-organizer classifier. Reply ONLY minified JSON: "
+              "{\"file_class\":\"..\",\"subject\":\"..\",\"importance\":\"..\",\"action\":\"..\",\"confidence\":0.0,\"rename\":\"..\"}\n"
+              "Examples:\n"
+              "Input: invoice_stripe_march.pdf (PDF, preview: 'Stripe payment receipt $120.00 subscription') -> "
+              "{\"file_class\":\"financial\",\"subject\":\"finance\",\"importance\":\"important\",\"action\":\"organize\",\"confidence\":0.97,\"rename\":\"2026-03_stripe_invoice.pdf\"}\n"
+              "Input: novel_draft_v2.docx (DOCX, preview: 'Chapter One. The lighthouse keeper woke...') -> "
+              "{\"file_class\":\"document\",\"subject\":\"personal\",\"importance\":\"important\",\"action\":\"organize\",\"confidence\":0.9,\"rename\":\"novel_draft_v2.docx\"}\n"
+              "Input: dumps_fullz_usa.txt (TXT, preview: 'CC number exp cvv ssn address...') -> "
+              "{\"file_class\":\"personal\",\"subject\":\"unknown\",\"importance\":\"critical\",\"action\":\"quarantine\",\"confidence\":0.95,\"rename\":\"dumps_fullz_usa.txt\"}\n"
+              "Now classify:\n"
               f"File: {row['name']} ({row['ext'] or 'no ext'}, {row['size']} bytes)\n"
               + (f"Content preview: {row['preview'][:400]}\n" if row['preview'] else "")
               + "file_class in " + str(ENUMS["file_class"]) + "; subject in " + str(ENUMS["subject"])
@@ -206,9 +283,10 @@ def classify_one(row):
               + "; confidence 0.0-1.0; rename = clean proposed filename or same")
     import urllib.request
     body = json.dumps({"model": MODEL, "prompt": prompt, "stream": False, "think": False,
-                       "options": {"temperature": 0.1, "num_predict": 400}}).encode()
+                       "keep_alive": -1,
+                       "options": {"temperature": 0.05, "num_predict": 220}}).encode()
     req = urllib.request.Request(OLLAMA + "/api/generate", data=body, headers={"Content-Type": "application/json"})
-    with urllib.request.urlopen(req, timeout=120) as r:
+    with urllib.request.urlopen(req, timeout=60) as r:
         d = json.loads(r.read())
     txt = d.get("response", "")
     m = re.search(r"\{.*\}", txt, re.S)
@@ -230,22 +308,51 @@ def classify_one(row):
     out["rename"] = re.sub(r"[^A-Za-z0-9 ._-]", "", rn)[:120] or row["name"]
     return out
 
+def classify_one_file(fid):
+    c = db()
+    row = c.execute("SELECT * FROM files WHERE id=?", (fid,)).fetchone()
+    if not row or (row["file_class"] and row["subject"]):
+        c.close(); return None
+    if row["preview"] is None:
+        prev = extract_preview(row["path"], row["ext"], row["size"])
+        c.execute("UPDATE files SET preview=? WHERE id=?", (prev, fid))
+        row = dict(row); row["preview"] = prev
+    try:
+        res = classify_one(row)
+    except Exception:
+        res = None
+    if not res:
+        # deterministic fallback — NEVER leave NULLs or stall the pipeline
+        qr = QUICK_RULES.get(row["ext"]) or ("unknown", "unknown", "normal", 0.3)
+        res = {"file_class": qr[0], "subject": qr[1] or "unknown", "importance": qr[2] or "normal",
+               "action": "review", "confidence": qr[3], "rename": row["name"]}
+    c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?",
+              (res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], fid))
+    c.commit()
+    c.close()
+    return res
+
 def classify_thread(limit):
+    from concurrent.futures import ThreadPoolExecutor, as_completed
     try:
         with LOCK: STATE["classify_running"] = True
         c = db()
-        rows = c.execute("SELECT * FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall()
+        fids = [r["id"] for r in c.execute("SELECT id FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall()]
-        STATE["classify_total"] = len(rows); STATE["classify_done"] = 0
+        c.close()
-        for row in rows:
+        STATE["classify_total"] = len(fids); STATE["classify_done"] = 0
-            res = classify_one(row)
+        # warm the model once (cold load can take 40s; warm calls ~4s)
-            if res:
+        try:
-                if row["preview"] is None:
+            import urllib.request
-                    prev = extract_preview(row["path"], row["ext"], row["size"])
+            body = json.dumps({"model": MODEL, "prompt": "warmup", "stream": False, "think": False,
-                    c.execute("UPDATE files SET preview=? WHERE id=?", (prev, row["id"]))
+                               "keep_alive": -1, "options": {"num_predict": 1}}).encode()
-                c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?",
+            urllib.request.urlopen(urllib.request.Request(OLLAMA + "/api/generate", data=body,
-                          (res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], row["id"]))
+                                   headers={"Content-Type": "application/json"}), timeout=90).read()
-            STATE["classify_done"] += 1
+        except Exception:
-        c.commit(); c.close()
+            pass
+        with ThreadPoolExecutor(max_workers=4) as ex:
+            futs = {ex.submit(classify_one_file, fid): fid for fid in fids}
+            for f in as_completed(futs):
+                STATE["classify_done"] += 1
     except Exception:
         STATE["last_error"] = traceback.format_exc()[-800:]
     finally:
@@ -462,7 +569,12 @@ def shell(body, active="home"):
             "<nav><div class='in'><span class='brand'>♎ LIBRA</span>" + nav + "</div></nav>"
             "<div class='wrap'>" + body + "</div>"
             "<div class='toast' id='toast'></div>"
-            "<script>window.LIBRA=" + json.dumps(STATE) + ";</script></body></html>")
+            "<script>window.LIBRA=" + json.dumps(STATE) + ";\n"
+            "(function pulse(){fetch('/api/state').then(function(r){return r.json()}).then(function(s){"
+            "var el=function(id){return document.getElementById(id)};"
+            "if(el('st-files')){el('st-files').textContent=s.files;el('st-classified').textContent=s.classified;"
+            "el('st-dupes').textContent=s.dupes;el('st-pending').textContent=s.pending;el('st-moved').textContent=s.moved}"
+            "window.LIBRA=s;setTimeout(pulse,2000)}).catch(function(){setTimeout(pulse,4000)})})();</script></body></html>")
 
 @app.route("/")
 def home():
@@ -473,11 +585,11 @@ def home():
     body = f"""
 <h1 class='grad'>File Librarian</h1><p class='muted'>Scanner → Extractor → Nimble → Safety → You approve → Reversible move. Nothing moves without your click.</p>
 <div class='grid'>
- <div class='card'><div class='n'>{STATE.get('files',0)}</div><div class='l'>Files indexed</div></div>
+ <div class='card'><div class='n' id='st-files'>{STATE.get('files',0)}</div><div class='l'>Files indexed</div></div>
- <div class='card'><div class='n'>{STATE.get('classified',0)}</div><div class='l'>Classified</div></div>
+ <div class='card'><div class='n' id='st-classified'>{STATE.get('classified',0)}</div><div class='l'>Classified</div></div>
- <div class='card'><div class='n'>{STATE.get('dupes',0)}</div><div class='l'>In dup groups</div></div>
+ <div class='card'><div class='n' id='st-dupes'>{STATE.get('dupes',0)}</div><div class='l'>In dup groups</div></div>
- <div class='card'><div class='n'>{STATE.get('pending',0)}</div><div class='l'>Approved (dry)</div></div>
+ <div class='card'><div class='n' id='st-pending'>{STATE.get('pending',0)}</div><div class='l'>Approved (dry)</div></div>
- <div class='card'><div class='n'>{STATE.get('moved',0)}</div><div class='l'>Moved</div></div>
+ <div class='card'><div class='n' id='st-moved'>{STATE.get('moved',0)}</div><div class='l'>Moved</div></div>
 </div>{err}
 <div class='panel'><h2>1 · Scan a location</h2>{scan_status}
 <form method='post' action='/scan'><div class='row'>
@@ -501,6 +613,11 @@ def home():
 """
     return Response(shell(body, "home"), mimetype="text/html")
 
+@app.route("/api/state")
+def api_state():
+    refresh_counts()
+    return {k: v for k, v in STATE.items() if isinstance(v, (int, float, bool, str))}
+
 @app.route("/scan", methods=["POST"])
 def scan_post():
     root = request.form.get("root", "~/Downloads")
@@ -593,7 +710,10 @@ function applyMoves(dry) {{
   fetch('/api/apply', {{ method:'POST', headers:{{'Content-Type':'application/json'}},
     body: JSON.stringify({{ ids: ids, dry_run: dry }}) }})
   .then(function(r){{ return r.json(); }})
-  .then(function(d) {{ document.getElementById('pmsg').textContent = 'done: ' + d.done + ' applied, ' + d.skipped + ' skipped, session ' + d.session; }});
+  .then(function(d) {{ var t=document.getElementById('toast'); t.textContent='✓ ' + d.done + ' applied · ' + d.skipped + ' skipped · session ' + d.session + (dry ? ' (dry run)' : ''); t.style.display='block'; setTimeout(function(){{t.style.display='none'}},5000);
+    document.getElementById('pmsg').textContent = t.textContent;
+    document.querySelectorAll('.pick').forEach(function(cb){{ cb.checked=false; }});
+  }});
 }}
 </script>
 """