v2: quick-rule fast path + content signals (deterministic first), parallel Nimble w/ warmup+keep_alive, live pulse stats, apply toast, self-healing db
This commit is contained in:
drjones
172
app.py
172
app.py
@@ -77,12 +77,61 @@ CREATE TABLE IF NOT EXISTS sessions(
|
||||
""")
|
||||
c.commit(); c.close()
|
||||
|
||||
# deterministic fast-path: obvious extensions never need the LLM (instant + accurate)
|
||||
QUICK_RULES = {
|
||||
".pdf": ("document", None, None, 0.75), ".doc": ("document", None, None, 0.75), ".docx": ("document", None, None, 0.75),
|
||||
".xlsx": ("document", "finance", None, 0.8), ".xls": ("document", "finance", None, 0.8), ".pptx": ("document", "work", None, 0.8),
|
||||
".txt": ("document", None, None, 0.6), ".md": ("document", None, None, 0.6), ".csv": ("dataset", None, None, 0.8),
|
||||
".jpg": ("image", "photography", None, 0.9), ".jpeg": ("image", "photography", None, 0.9), ".png": ("image", None, None, 0.85),
|
||||
".heic": ("image", "photography", None, 0.9), ".gif": ("image", None, None, 0.85), ".webp": ("image", None, None, 0.85),
|
||||
".psd": ("image", None, None, 0.8),
|
||||
".mp4": ("video", None, None, 0.9), ".mov": ("video", None, None, 0.9), ".mkv": ("video", None, None, 0.9), ".avi": ("video", None, None, 0.9),
|
||||
".mp3": ("audio", None, None, 0.9), ".wav": ("audio", None, None, 0.9), ".flac": ("audio", None, None, 0.9),
|
||||
".zip": ("archive", None, None, 0.85), ".rar": ("archive", None, None, 0.85), ".7z": ("archive", None, None, 0.85),
|
||||
".tar": ("archive", None, None, 0.85), ".gz": ("archive", None, None, 0.85), ".iso": ("archive", None, None, 0.8),
|
||||
".py": ("source_code", "programming", None, 0.92), ".js": ("source_code", "programming", None, 0.92), ".ts": ("source_code", "programming", None, 0.92),
|
||||
".sh": ("source_code", "linux", None, 0.9), ".go": ("source_code", "programming", None, 0.92), ".rs": ("source_code", "programming", None, 0.92),
|
||||
".c": ("source_code", "programming", None, 0.9), ".cpp": ("source_code", "programming", None, 0.9), ".java": ("source_code", "programming", None, 0.92),
|
||||
".exe": ("software", None, None, 0.9), ".msi": ("software", None, None, 0.9), ".dmg": ("software", None, None, 0.9), ".app": ("software", None, None, 0.9),
|
||||
".deb": ("software", "linux", None, 0.9), ".rpm": ("software", "linux", None, 0.9), ".apk": ("software", None, None, 0.9),
|
||||
".ova": ("dataset", None, None, 0.7), ".vmdk": ("dataset", None, None, 0.7), ".sqlite": ("dataset", None, None, 0.8), ".db": ("dataset", None, None, 0.8),
|
||||
".epub": ("ebook", None, None, 0.95), ".mobi": ("ebook", None, None, 0.95), ".azw3": ("ebook", None, None, 0.95), ".djvu": ("ebook", None, None, 0.9),
|
||||
}
|
||||
# exts that LOOK like ebooks/docs but need content sniffing (pdf can be either) — handled by classifier
|
||||
|
||||
# content signals — deterministic, beat quick rules when the content screams (receipt, fullz, hacking…)
|
||||
SIGNALS = [
|
||||
(r"receipt|invoice|order confirmation|statement|payment|charged \$|billing", "financial", "finance", "important", "organize", 0.97),
|
||||
(r"cvv|fullz|ssn|card number|carding|dump[s]? ", "personal", "unknown", "critical", "quarantine", 0.95),
|
||||
(r"hacking|exploit|keylogger|rat |botnet|malware", "document", "cybersecurity", "important", "quarantine", 0.93),
|
||||
(r"chapter one|novel|fiction|once upon", "document", "personal", "important", "organize", 0.9),
|
||||
]
|
||||
|
||||
def sniff_content(p, ext, size):
|
||||
if ext not in (".txt", ".md", ".csv", ".json", ".log", ".pdf", ".html", ".cfg", ".ini") or size > 5_000_000:
|
||||
return None
|
||||
try:
|
||||
if ext == ".pdf":
|
||||
from pypdf import PdfReader
|
||||
txt = " ".join((pg.extract_text() or "") for pg in PdfReader(p).pages[:2])
|
||||
else:
|
||||
txt = open(p, "r", errors="replace").read(4000)
|
||||
except Exception:
|
||||
return None
|
||||
low = txt.lower()
|
||||
for rx, fc, sub, imp, act, conf in SIGNALS:
|
||||
if re.search(rx, low):
|
||||
return {"file_class": fc, "subject": sub, "importance": imp, "action": act,
|
||||
"confidence": conf, "rename": None, "preview": re.sub(r"\s+", " ", txt)[:600]}
|
||||
return None
|
||||
|
||||
# ---------- SCANNER ----------
|
||||
SKIP_DIRS = {".Trashes", ".Spotlight-V100", ".TemporaryItems", ".fseventsd", "System Volume Information",
|
||||
"$RECYCLE.BIN", "node_modules", "__pycache__", ".git", "Library", ".Trash"}
|
||||
JUNK = (".DS_Store",)
|
||||
|
||||
def scan(root, max_files=5000, min_size=0, max_size=0):
|
||||
init_db()
|
||||
root = os.path.abspath(os.path.expanduser(root))
|
||||
c = db()
|
||||
n = 0
|
||||
@@ -99,6 +148,22 @@ def scan(root, max_files=5000, min_size=0, max_size=0):
|
||||
if st.st_size < min_size or (max_size and st.st_size > max_size):
|
||||
continue
|
||||
try:
|
||||
qr = QUICK_RULES.get(os.path.splitext(f)[1].lower())
|
||||
if qr:
|
||||
fc, sub, imp, conf = qr
|
||||
sig = sniff_content(p, os.path.splitext(f)[1].lower(), st.st_size)
|
||||
if sig:
|
||||
fc, sub, imp, conf = sig["file_class"], sig["subject"], sig["importance"], sig["confidence"]
|
||||
c.execute("""INSERT OR IGNORE INTO files(path,name,ext,size,mtime,file_class,subject,importance,confidence,action,preview,status)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?, 'classified')""",
|
||||
(p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime, fc, sub, imp, conf, sig["action"], sig["preview"]))
|
||||
n += 1
|
||||
continue
|
||||
c.execute("""INSERT OR IGNORE INTO files(path,name,ext,size,mtime,file_class,subject,importance,confidence,action,status)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,'organize','classified')""",
|
||||
(p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime, fc, sub, imp or 'normal', conf))
|
||||
n += 1
|
||||
continue
|
||||
c.execute("INSERT OR IGNORE INTO files(path,name,ext,size,mtime,status) VALUES(?,?,?,?,?, 'scanned')",
|
||||
(p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime))
|
||||
except sqlite3.Error:
|
||||
@@ -124,15 +189,18 @@ def scan_thread(root, max_files, min_size, max_size):
|
||||
refresh_counts()
|
||||
|
||||
def refresh_counts():
|
||||
c = db()
|
||||
for k, q in [("files", "SELECT COUNT(*) FROM files"),
|
||||
try:
|
||||
c = db()
|
||||
for k, q in [("files", "SELECT COUNT(*) FROM files"),
|
||||
("scanned", "SELECT COUNT(*) FROM files WHERE file_class IS NULL"),
|
||||
("classified", "SELECT COUNT(*) FROM files WHERE file_class IS NOT NULL"),
|
||||
("dupes", "SELECT COUNT(*) FROM files WHERE sha256 IS NOT NULL AND sha256 IN (SELECT sha256 FROM files WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(*)>1)"),
|
||||
("pending", "SELECT COUNT(*) FROM files WHERE status='approved'"),
|
||||
("moved", "SELECT COUNT(*) FROM files WHERE status='moved'")]:
|
||||
STATE[k] = c.execute(q).fetchone()[0]
|
||||
c.close()
|
||||
STATE[k] = c.execute(q).fetchone()[0]
|
||||
c.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ---------- EXTRACTOR ----------
|
||||
def extract_preview(p, ext, size):
|
||||
@@ -198,7 +266,16 @@ ENUMS = {
|
||||
"action": ["keep","organize","archive","quarantine","review"],
|
||||
}
|
||||
def classify_one(row):
|
||||
prompt = ("Classify this file. Reply ONLY minified JSON with keys file_class, subject, importance, action, confidence, rename. No other text.\n"
|
||||
prompt = ("You are a file-organizer classifier. Reply ONLY minified JSON: "
|
||||
"{\"file_class\":\"..\",\"subject\":\"..\",\"importance\":\"..\",\"action\":\"..\",\"confidence\":0.0,\"rename\":\"..\"}\n"
|
||||
"Examples:\n"
|
||||
"Input: invoice_stripe_march.pdf (PDF, preview: 'Stripe payment receipt $120.00 subscription') -> "
|
||||
"{\"file_class\":\"financial\",\"subject\":\"finance\",\"importance\":\"important\",\"action\":\"organize\",\"confidence\":0.97,\"rename\":\"2026-03_stripe_invoice.pdf\"}\n"
|
||||
"Input: novel_draft_v2.docx (DOCX, preview: 'Chapter One. The lighthouse keeper woke...') -> "
|
||||
"{\"file_class\":\"document\",\"subject\":\"personal\",\"importance\":\"important\",\"action\":\"organize\",\"confidence\":0.9,\"rename\":\"novel_draft_v2.docx\"}\n"
|
||||
"Input: dumps_fullz_usa.txt (TXT, preview: 'CC number exp cvv ssn address...') -> "
|
||||
"{\"file_class\":\"personal\",\"subject\":\"unknown\",\"importance\":\"critical\",\"action\":\"quarantine\",\"confidence\":0.95,\"rename\":\"dumps_fullz_usa.txt\"}\n"
|
||||
"Now classify:\n"
|
||||
f"File: {row['name']} ({row['ext'] or 'no ext'}, {row['size']} bytes)\n"
|
||||
+ (f"Content preview: {row['preview'][:400]}\n" if row['preview'] else "")
|
||||
+ "file_class in " + str(ENUMS["file_class"]) + "; subject in " + str(ENUMS["subject"])
|
||||
@@ -206,9 +283,10 @@ def classify_one(row):
|
||||
+ "; confidence 0.0-1.0; rename = clean proposed filename or same")
|
||||
import urllib.request
|
||||
body = json.dumps({"model": MODEL, "prompt": prompt, "stream": False, "think": False,
|
||||
"options": {"temperature": 0.1, "num_predict": 400}}).encode()
|
||||
"keep_alive": -1,
|
||||
"options": {"temperature": 0.05, "num_predict": 220}}).encode()
|
||||
req = urllib.request.Request(OLLAMA + "/api/generate", data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
d = json.loads(r.read())
|
||||
txt = d.get("response", "")
|
||||
m = re.search(r"\{.*\}", txt, re.S)
|
||||
@@ -230,22 +308,51 @@ def classify_one(row):
|
||||
out["rename"] = re.sub(r"[^A-Za-z0-9 ._-]", "", rn)[:120] or row["name"]
|
||||
return out
|
||||
|
||||
def classify_one_file(fid):
|
||||
c = db()
|
||||
row = c.execute("SELECT * FROM files WHERE id=?", (fid,)).fetchone()
|
||||
if not row or (row["file_class"] and row["subject"]):
|
||||
c.close(); return None
|
||||
if row["preview"] is None:
|
||||
prev = extract_preview(row["path"], row["ext"], row["size"])
|
||||
c.execute("UPDATE files SET preview=? WHERE id=?", (prev, fid))
|
||||
row = dict(row); row["preview"] = prev
|
||||
try:
|
||||
res = classify_one(row)
|
||||
except Exception:
|
||||
res = None
|
||||
if not res:
|
||||
# deterministic fallback — NEVER leave NULLs or stall the pipeline
|
||||
qr = QUICK_RULES.get(row["ext"]) or ("unknown", "unknown", "normal", 0.3)
|
||||
res = {"file_class": qr[0], "subject": qr[1] or "unknown", "importance": qr[2] or "normal",
|
||||
"action": "review", "confidence": qr[3], "rename": row["name"]}
|
||||
c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?",
|
||||
(res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], fid))
|
||||
c.commit()
|
||||
c.close()
|
||||
return res
|
||||
|
||||
def classify_thread(limit):
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
try:
|
||||
with LOCK: STATE["classify_running"] = True
|
||||
c = db()
|
||||
rows = c.execute("SELECT * FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall()
|
||||
STATE["classify_total"] = len(rows); STATE["classify_done"] = 0
|
||||
for row in rows:
|
||||
res = classify_one(row)
|
||||
if res:
|
||||
if row["preview"] is None:
|
||||
prev = extract_preview(row["path"], row["ext"], row["size"])
|
||||
c.execute("UPDATE files SET preview=? WHERE id=?", (prev, row["id"]))
|
||||
c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?",
|
||||
(res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], row["id"]))
|
||||
STATE["classify_done"] += 1
|
||||
c.commit(); c.close()
|
||||
fids = [r["id"] for r in c.execute("SELECT id FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall()]
|
||||
c.close()
|
||||
STATE["classify_total"] = len(fids); STATE["classify_done"] = 0
|
||||
# warm the model once (cold load can take 40s; warm calls ~4s)
|
||||
try:
|
||||
import urllib.request
|
||||
body = json.dumps({"model": MODEL, "prompt": "warmup", "stream": False, "think": False,
|
||||
"keep_alive": -1, "options": {"num_predict": 1}}).encode()
|
||||
urllib.request.urlopen(urllib.request.Request(OLLAMA + "/api/generate", data=body,
|
||||
headers={"Content-Type": "application/json"}), timeout=90).read()
|
||||
except Exception:
|
||||
pass
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
futs = {ex.submit(classify_one_file, fid): fid for fid in fids}
|
||||
for f in as_completed(futs):
|
||||
STATE["classify_done"] += 1
|
||||
except Exception:
|
||||
STATE["last_error"] = traceback.format_exc()[-800:]
|
||||
finally:
|
||||
@@ -462,7 +569,12 @@ def shell(body, active="home"):
|
||||
"<nav><div class='in'><span class='brand'>♎ LIBRA</span>" + nav + "</div></nav>"
|
||||
"<div class='wrap'>" + body + "</div>"
|
||||
"<div class='toast' id='toast'></div>"
|
||||
"<script>window.LIBRA=" + json.dumps(STATE) + ";</script></body></html>")
|
||||
"<script>window.LIBRA=" + json.dumps(STATE) + ";\n"
|
||||
"(function pulse(){fetch('/api/state').then(function(r){return r.json()}).then(function(s){"
|
||||
"var el=function(id){return document.getElementById(id)};"
|
||||
"if(el('st-files')){el('st-files').textContent=s.files;el('st-classified').textContent=s.classified;"
|
||||
"el('st-dupes').textContent=s.dupes;el('st-pending').textContent=s.pending;el('st-moved').textContent=s.moved}"
|
||||
"window.LIBRA=s;setTimeout(pulse,2000)}).catch(function(){setTimeout(pulse,4000)})})();</script></body></html>")
|
||||
|
||||
@app.route("/")
|
||||
def home():
|
||||
@@ -473,11 +585,11 @@ def home():
|
||||
body = f"""
|
||||
<h1 class='grad'>File Librarian</h1><p class='muted'>Scanner → Extractor → Nimble → Safety → You approve → Reversible move. Nothing moves without your click.</p>
|
||||
<div class='grid'>
|
||||
<div class='card'><div class='n'>{STATE.get('files',0)}</div><div class='l'>Files indexed</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('classified',0)}</div><div class='l'>Classified</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('dupes',0)}</div><div class='l'>In dup groups</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('pending',0)}</div><div class='l'>Approved (dry)</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('moved',0)}</div><div class='l'>Moved</div></div>
|
||||
<div class='card'><div class='n' id='st-files'>{STATE.get('files',0)}</div><div class='l'>Files indexed</div></div>
|
||||
<div class='card'><div class='n' id='st-classified'>{STATE.get('classified',0)}</div><div class='l'>Classified</div></div>
|
||||
<div class='card'><div class='n' id='st-dupes'>{STATE.get('dupes',0)}</div><div class='l'>In dup groups</div></div>
|
||||
<div class='card'><div class='n' id='st-pending'>{STATE.get('pending',0)}</div><div class='l'>Approved (dry)</div></div>
|
||||
<div class='card'><div class='n' id='st-moved'>{STATE.get('moved',0)}</div><div class='l'>Moved</div></div>
|
||||
</div>{err}
|
||||
<div class='panel'><h2>1 · Scan a location</h2>{scan_status}
|
||||
<form method='post' action='/scan'><div class='row'>
|
||||
@@ -501,6 +613,11 @@ def home():
|
||||
"""
|
||||
return Response(shell(body, "home"), mimetype="text/html")
|
||||
|
||||
@app.route("/api/state")
|
||||
def api_state():
|
||||
refresh_counts()
|
||||
return {k: v for k, v in STATE.items() if isinstance(v, (int, float, bool, str))}
|
||||
|
||||
@app.route("/scan", methods=["POST"])
|
||||
def scan_post():
|
||||
root = request.form.get("root", "~/Downloads")
|
||||
@@ -593,7 +710,10 @@ function applyMoves(dry) {{
|
||||
fetch('/api/apply', {{ method:'POST', headers:{{'Content-Type':'application/json'}},
|
||||
body: JSON.stringify({{ ids: ids, dry_run: dry }}) }})
|
||||
.then(function(r){{ return r.json(); }})
|
||||
.then(function(d) {{ document.getElementById('pmsg').textContent = 'done: ' + d.done + ' applied, ' + d.skipped + ' skipped, session ' + d.session; }});
|
||||
.then(function(d) {{ var t=document.getElementById('toast'); t.textContent='✓ ' + d.done + ' applied · ' + d.skipped + ' skipped · session ' + d.session + (dry ? ' (dry run)' : ''); t.style.display='block'; setTimeout(function(){{t.style.display='none'}},5000);
|
||||
document.getElementById('pmsg').textContent = t.textContent;
|
||||
document.querySelectorAll('.pick').forEach(function(cb){{ cb.checked=false; }});
|
||||
}});
|
||||
}}
|
||||
</script>
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user