From 4e38f7257e74595a224d520f21f85afb6f4d21d1 Mon Sep 17 00:00:00 2001 From: drjones Date: Sun, 4 Oct 2026 22:05:12 -0700 Subject: [PATCH] LIBRA v1: scanner/extractor/Nimble classifier/safety gate/approval UI/op log/undo/dupe hold --- .gitignore | 4 + README.md | 26 ++ app.py | 712 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 + taxonomy.yaml | 31 +++ 5 files changed, 776 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 app.py create mode 100644 requirements.txt create mode 100644 taxonomy.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2152be7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +libra.db +dupe_hold/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..0f1eee9 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# LIBRA ♎ — the Scales. Nimble-classified file librarian. + +Nimble is the classification brain; deterministic Python touches the filesystem. +Nothing moves without your click. Every move is logged and one-click reversible. + +## Pipeline +scan → sha256 hash (deterministic dup detection) → content extract (pdf/zip/text/images) → +Nimble structured JSON classification (think:false, strict enums) → safety validate → +review plan UI → apply (dry-run default) → immutable op log → undo session. + +## Run +``` +python3 ~/libra/app.py # http://127.0.0.1:5731 +``` +Requires: flask, pypdf (optional: Pillow). Nimble served by Ollama on nightmare (10.30.20.29:11434, model nimble:latest). + +## Safety gates +- No system paths, no symlinks, no overwrite (auto-suffix). +- Confidence gate (tunable) — below threshold → REVIEW. +- Duplicates are sha256-verified only; extras go to ~/libra/dupe_hold/, never deleted. +- Every operation logged with src/dst/sha256; sessions undoable. + +## Layout +- app.py — everything (scanner, extractor, classifier, safety, organizer, UI) +- taxonomy.yaml — editable destination map (class+subject → folder) +- libra.db — SQLite index of files, operations, sessions diff --git a/app.py b/app.py new file mode 100644 index 0000000..4bba324 --- /dev/null +++ b/app.py @@ -0,0 +1,712 @@ +#!/usr/bin/env python3 +"""LIBRA — the Scales. Nimble-classified file organizer with approval-gated, reversible moves. +Local only: Flask on Mac, Ollama Nimble decision lane on nightmare .29.""" +import os, re, json, time, uuid, hashlib, sqlite3, threading, shutil, traceback, fnmatch +import subprocess +from pathlib import Path +from flask import Flask, request, redirect, Response + +APP_DIR = os.path.expanduser("~/libra") +DB_PATH = os.path.join(APP_DIR, "libra.db") +CFG_PATH = os.path.join(APP_DIR, "taxonomy.yaml") + +OLLAMA = "http://10.30.20.29:11434" +MODEL = "nimble:latest" +HOME = os.path.expanduser("~") + +DEFAULT_TAXONOMY = """# Libra taxonomy — destination map (editable here) +# mapping: '+' -> destination (relative to organized root) +organized_root: ~/libra-organized +map: + document+finance: Documents/Financial + document+work: Documents/Work + document+personal: Documents/Personal + document+unknown: Documents/Misc + ebook+reference: Knowledge/Books + ebook+programming: Knowledge/Books + ebook+AI: Knowledge/Books + ebook+linux: Knowledge/Books + ebook+cybersecurity: Knowledge/Books + ebook+unknown: Knowledge/Books + ebook+personal: Knowledge/Books + image+photography: Media/Photos + image+personal: Media/Photos + image+unknown: Media/Photos + video+unknown: Media/Videos + audio+unknown: Media/Audio + software+unknown: Software/Installers + software+programming: Software/Source + source_code+programming: Software/Source + source_code+AI: Software/Source + archive+unknown: Data/Archives + dataset+unknown: Data/Datasets + financial+finance: Documents/Financial + personal+personal: Documents/Personal + temporary+unknown: Unsorted + unknown+unknown: Unsorted +default: Unsorted +""" + +if not os.path.exists(CFG_PATH): + Path(APP_DIR).mkdir(parents=True, exist_ok=True) + open(CFG_PATH, "w").write(DEFAULT_TAXONOMY) + +app = Flask(__name__) +LOCK = threading.Lock() +STATE = {"scan_running": False, "scan_done": 0, "scan_total": 0, "classify_running": False, + "classify_done": 0, "classify_total": 0, "last_error": ""} + +# ---------- DB ---------- +def db(): + c = sqlite3.connect(DB_PATH, timeout=30) + c.row_factory = sqlite3.Row + return c + +def init_db(): + c = db() + c.executescript(""" +CREATE TABLE IF NOT EXISTS files( + id INTEGER PRIMARY KEY, path TEXT UNIQUE, name TEXT, ext TEXT, size INTEGER, mtime REAL, + sha256 TEXT, preview TEXT, file_class TEXT, subject TEXT, importance TEXT, action TEXT, + confidence REAL, rename TEXT, status TEXT DEFAULT 'scanned', reason TEXT); +CREATE TABLE IF NOT EXISTS operations( + id INTEGER PRIMARY KEY, ts REAL, session TEXT, file_id INTEGER, op TEXT, + src TEXT, dst TEXT, sha256 TEXT, undone INTEGER DEFAULT 0); +CREATE TABLE IF NOT EXISTS sessions( + id TEXT PRIMARY KEY, ts REAL, n INTEGER, undone INTEGER DEFAULT 0, note TEXT); +""") + c.commit(); c.close() + +# ---------- SCANNER ---------- +SKIP_DIRS = {".Trashes", ".Spotlight-V100", ".TemporaryItems", ".fseventsd", "System Volume Information", + "$RECYCLE.BIN", "node_modules", "__pycache__", ".git", "Library", ".Trash"} +JUNK = (".DS_Store",) + +def scan(root, max_files=5000, min_size=0, max_size=0): + root = os.path.abspath(os.path.expanduser(root)) + c = db() + n = 0 + for r, dirs, fs in os.walk(root): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith("._")] + for f in fs: + if f in JUNK or f.startswith("._"): + continue + p = os.path.join(r, f) + try: + st = os.lstat(p) + except OSError: + continue + if st.st_size < min_size or (max_size and st.st_size > max_size): + continue + try: + c.execute("INSERT OR IGNORE INTO files(path,name,ext,size,mtime,status) VALUES(?,?,?,?,?, 'scanned')", + (p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime)) + except sqlite3.Error: + pass + n += 1 + if n >= max_files: + break + if n >= max_files: + break + c.commit(); c.close() + return n + +def scan_thread(root, max_files, min_size, max_size): + try: + with LOCK: + STATE["scan_running"] = True + scan(root, max_files, min_size, max_size) + except Exception: + STATE["last_error"] = traceback.format_exc()[-800:] + finally: + with LOCK: + STATE["scan_running"] = False + refresh_counts() + +def refresh_counts(): + c = db() + for k, q in [("files", "SELECT COUNT(*) FROM files"), + ("scanned", "SELECT COUNT(*) FROM files WHERE file_class IS NULL"), + ("classified", "SELECT COUNT(*) FROM files WHERE file_class IS NOT NULL"), + ("dupes", "SELECT COUNT(*) FROM files WHERE sha256 IS NOT NULL AND sha256 IN (SELECT sha256 FROM files WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(*)>1)"), + ("pending", "SELECT COUNT(*) FROM files WHERE status='approved'"), + ("moved", "SELECT COUNT(*) FROM files WHERE status='moved'")]: + STATE[k] = c.execute(q).fetchone()[0] + c.close() + +# ---------- EXTRACTOR ---------- +def extract_preview(p, ext, size): + prev = "" + try: + if ext == ".pdf" and size < 60_000_000: + from pypdf import PdfReader + rd = PdfReader(p) + txt = "" + for pg in rd.pages[:2]: + txt += (pg.extract_text() or "") + " " + prev = re.sub(r"\s+", " ", txt)[:600] + elif ext in (".txt", ".md", ".csv", ".json", ".yaml", ".yml", ".log", ".py", ".sh", ".html", ".cfg", ".ini") and size < 5_000_000: + prev = open(p, "r", errors="replace").read(600) + elif ext in (".zip",) and size < 500_000_000: + import zipfile + with zipfile.ZipFile(p) as z: + names = z.namelist()[:20] + prev = "zip listing: " + ", ".join(names) + elif ext in (".jpg", ".jpeg", ".png", ".heic", ".gif", ".webp"): + try: + from PIL import Image + im = Image.open(p) + prev = f"image {im.size[0]}x{im.size[1]}" + except Exception: + prev = "image file" + except Exception: + prev = "" + return prev + +def hash_files(limit=500, max_mb=64): + c = db() + rows = c.execute("SELECT id,path,size FROM files WHERE sha256 IS NULL LIMIT ?", (limit,)).fetchall() + for row in rows: + if row["size"] > max_mb * 1_000_000: + c.execute("UPDATE files SET sha256='TOOBIG' WHERE id=?", (row["id"],)); continue + h = hashlib.sha256() + try: + with open(row["path"], "rb") as fh: + for chunk in iter(lambda: fh.read(4 << 20), b""): + h.update(chunk) + c.execute("UPDATE files SET sha256=? WHERE id=?", (h.hexdigest(), row["id"])) + except OSError: + c.execute("UPDATE files SET sha256='UNREADABLE' WHERE id=?", (row["id"],)) + c.commit(); n = len(rows); c.close(); return n + +def hash_thread(limit, max_mb): + try: + with LOCK: STATE["classify_running"] = True + while hash_files(limit, max_mb): + time.sleep(0.1) + except Exception: + STATE["last_error"] = traceback.format_exc()[-800:] + finally: + with LOCK: STATE["classify_running"] = False + refresh_counts() + +# ---------- CLASSIFIER (Nimble) ---------- +ENUMS = { + "file_class": ["document","image","video","audio","software","source_code","archive","dataset","ebook","financial","personal","temporary","unknown"], + "subject": ["work","finance","programming","AI","linux","cybersecurity","photography","personal","entertainment","reference","unknown"], + "importance": ["disposable","low","normal","important","critical"], + "action": ["keep","organize","archive","quarantine","review"], +} +def classify_one(row): + prompt = ("Classify this file. Reply ONLY minified JSON with keys file_class, subject, importance, action, confidence, rename. No other text.\n" + f"File: {row['name']} ({row['ext'] or 'no ext'}, {row['size']} bytes)\n" + + (f"Content preview: {row['preview'][:400]}\n" if row['preview'] else "") + + "file_class in " + str(ENUMS["file_class"]) + "; subject in " + str(ENUMS["subject"]) + + "; importance in " + str(ENUMS["importance"]) + "; action in " + str(ENUMS["action"]) + + "; confidence 0.0-1.0; rename = clean proposed filename or same") + import urllib.request + body = json.dumps({"model": MODEL, "prompt": prompt, "stream": False, "think": False, + "options": {"temperature": 0.1, "num_predict": 400}}).encode() + req = urllib.request.Request(OLLAMA + "/api/generate", data=body, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=120) as r: + d = json.loads(r.read()) + txt = d.get("response", "") + m = re.search(r"\{.*\}", txt, re.S) + if not m: + return None + try: + j = json.loads(m.group(0)) + except Exception: + return None + out = {} + for k, vals in ENUMS.items(): + v = str(j.get(k, "unknown")).strip().lower() + if v not in vals: v = "unknown" if k != "importance" else "normal" + out[k] = v + try: conf = float(j.get("confidence", 0.5)) + except Exception: conf = 0.5 + out["confidence"] = max(0.0, min(1.0, conf if conf <= 1.0 else conf / 10.0)) + rn = str(j.get("rename", "")).strip() + out["rename"] = re.sub(r"[^A-Za-z0-9 ._-]", "", rn)[:120] or row["name"] + return out + +def classify_thread(limit): + try: + with LOCK: STATE["classify_running"] = True + c = db() + rows = c.execute("SELECT * FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall() + STATE["classify_total"] = len(rows); STATE["classify_done"] = 0 + for row in rows: + res = classify_one(row) + if res: + if row["preview"] is None: + prev = extract_preview(row["path"], row["ext"], row["size"]) + c.execute("UPDATE files SET preview=? WHERE id=?", (prev, row["id"])) + c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?", + (res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], row["id"])) + STATE["classify_done"] += 1 + c.commit(); c.close() + except Exception: + STATE["last_error"] = traceback.format_exc()[-800:] + finally: + with LOCK: STATE["classify_running"] = False + refresh_counts() + +# ---------- TAXONOMY + SAFETY ---------- +def load_taxonomy(): + tx = {"organized_root": os.path.expanduser("~/libra-organized"), "map": {}, "default": "Unsorted"} + cur = None + for line in open(CFG_PATH): + line = line.rstrip() + if not line or line.lstrip().startswith("#"): continue + if line.startswith("organized_root:"): + tx["organized_root"] = os.path.expanduser(line.split(":", 1)[1].strip()) + elif line.startswith("default:"): + tx["default"] = line.split(":", 1)[1].strip() + elif line.startswith("map:"): + cur = "map" + elif cur == "map" and line.startswith(" ") and ":" in line: + k, v = line.strip().split(":", 1) + tx["map"][k.strip()] = v.strip() + return tx + +def propose_destination(row, tx): + key = f"{row['file_class']}+{row['subject']}" + sub = tx["map"].get(key, tx["map"].get(f"{row['file_class']}+unknown", tx["default"])) + date = time.strftime("%Y/%m", time.localtime(row["mtime"] or time.time())) + return os.path.join(tx["organized_root"], sub, date) + +def validate(src, dst, confidence, min_conf, tx): + if confidence < min_conf: return "REVIEW", "confidence below threshold" + if not src.startswith(tx.get("scan_root", "/")): return "REJECT", "outside scan root" + for sysd in ("/System", "/usr", "/etc", "/bin", "/sbin", "/var", "/private", "/opt"): + if src.startswith(sysd + "/"): return "REJECT", "system path" + if os.path.islink(src): return "REJECT", "symlink" + if os.path.exists(dst): return "REVIEW", "destination exists" + if os.path.dirname(src) == dst: return "REJECT", "already there" + return "APPROVE", "" + +def build_plan(tx, min_conf): + c = db() + rows = c.execute("SELECT * FROM files WHERE status='classified'").fetchall() + c.close() + dupset = set() + c = db() + for (s,) in c.execute("SELECT sha256 FROM files WHERE sha256 IS NOT NULL AND sha256 NOT IN ('TOOBIG','UNREADABLE') GROUP BY sha256 HAVING COUNT(*)>1"): + dupset.add(s) + c.close() + plans = [] + for row in rows: + if row["action"] not in ("organize", "archive"): continue + dst_dir = propose_destination(row, tx) + name = row["rename"] if row["rename"] else row["name"] + base, ext = os.path.splitext(name) + if ext.lower() != row["ext"] and row["ext"]: + name = base + row["ext"] + dst = os.path.join(dst_dir, name) + verdict, why = validate(row["path"], dst, row["confidence"], min_conf, tx) + is_dup = row["sha256"] in dupset + plans.append({"id": row["id"], "path": row["path"], "name": row["name"], "dst": dst, + "cls": row["file_class"], "subj": row["subject"], "imp": row["importance"], + "conf": row["confidence"], "verdict": verdict, "why": why, "dup": is_dup, + "size": row["size"]}) + return plans + +# ---------- ORGANIZER (apply / undo) ---------- +def apply_moves(ids, dry_run=True): + tx = load_taxonomy() + c = db() + session = uuid.uuid4().hex[:12] + done, skipped = 0, 0 + for fid in ids: + row = c.execute("SELECT * FROM files WHERE id=?", (fid,)).fetchone() + if not row or row["status"] not in ("classified", "approved"): skipped += 1; continue + dst_dir = propose_destination(row, tx) + name = row["rename"] or row["name"] + base, ext = os.path.splitext(name) + if ext.lower() != row["ext"] and row["ext"]: + name = base + row["ext"] + dst = os.path.join(dst_dir, name) + i = 1 + while os.path.exists(dst): + dst = os.path.join(dst_dir, f"{base}({i}){ext}"); i += 1 + verdict, why = validate(row["path"], dst, row["confidence"], 0.0, tx) + if verdict == "REJECT": + skipped += 1; continue + if dry_run: + c.execute("UPDATE files SET status='approved' WHERE id=?", (fid,)) + done += 1; continue + try: + os.makedirs(dst_dir, exist_ok=True) + shutil.move(row["path"], dst) + c.execute("INSERT INTO operations(ts,session,file_id,op,src,dst,sha256) VALUES(?,?,?,?,?,?,?)", + (time.time(), session, fid, "move", row["path"], dst, row["sha256"] or "")) + c.execute("UPDATE files SET path=?,name=?,status='moved' WHERE id=?", (dst, os.path.basename(dst), fid)) + done += 1 + except OSError as e: + skipped += 1 + STATE["last_error"] = f"{e}" + if not dry_run: + c.execute("INSERT INTO sessions(id,ts,n,note) VALUES(?,?,?,?)", (session, time.time(), done, "apply")) + c.commit(); c.close(); refresh_counts() + return done, skipped, session + +def undo_session(session): + c = db() + ops = c.execute("SELECT * FROM operations WHERE session=? AND undone=0 ORDER BY id DESC", (session,)).fetchall() + n = 0 + for op in ops: + try: + if os.path.exists(op["dst"]): + os.makedirs(os.path.dirname(op["src"]), exist_ok=True) + shutil.move(op["dst"], op["src"]) + c.execute("UPDATE files SET path=?,name=?,status='classified' WHERE id=?", (op["src"], os.path.basename(op["src"]), op["file_id"])) + c.execute("UPDATE operations SET undone=1 WHERE id=?", (op["id"],)) + n += 1 + except OSError as e: + STATE["last_error"] = str(e) + c.execute("UPDATE sessions SET undone=1 WHERE id=?", (session,)) + c.commit(); c.close(); refresh_counts() + return n + +def delete_duplicates(keep_id): + c = db() + row = c.execute("SELECT sha256 FROM files WHERE id=?", (keep_id,)).fetchone() + if not row or not row["sha256"]: c.close(); return 0 + session = uuid.uuid4().hex[:12] + n = 0 + for d in c.execute("SELECT * FROM files WHERE sha256=? AND id!=? AND status!='moved'", (row["sha256"], keep_id)).fetchall(): + try: + hold = os.path.join(APP_DIR, "dupe_hold", session) + os.makedirs(hold, exist_ok=True) + shutil.move(d["path"], os.path.join(hold, f"{n:04d}__" + d["name"])) + c.execute("INSERT INTO operations(ts,session,file_id,op,src,dst,sha256) VALUES(?,?,?,?,?,?,?)", + (time.time(), session, d["id"], "dupe_hold", d["path"], os.path.join(hold, d["name"]), d["sha256"])) + c.execute("UPDATE files SET status='held' WHERE id=?", (d["id"],)) + n += 1 + except OSError: + pass + c.execute("INSERT INTO sessions(id,ts,n,note) VALUES(?,?,?,?)", (session, time.time(), n, "duplicates to hold")) + c.commit(); c.close(); refresh_counts() + return n + +# ---------- UI ---------- +BASE_CSS = """ +:root{--bg:#0a0e14;--panel:rgba(20,26,38,.78);--line:rgba(120,160,255,.14);--txt:#dbe4f5;--dim:#8194b8;--acc:#7c6cff;--acc2:#35d0ba;--warn:#ffb454;--bad:#ff6b6b} +*{box-sizing:border-box;margin:0} +body{color:var(--txt);font:15px/1.55 -apple-system,'SF Pro Text',Helvetica,sans-serif;min-height:100vh; + background:linear-gradient(180deg,#0a0e14 0%,#0d1320 60%,#0a0e14 100%) fixed; + position:relative} +body:before{content:'';position:fixed;inset:0;pointer-events:none;z-index:0; + background:radial-gradient(900px 500px at 15% -10%,rgba(124,108,255,.08),transparent 60%), + radial-gradient(800px 500px at 90% 20%,rgba(53,208,186,.06),transparent 60%), + radial-gradient(700px 400px at 50% 110%,rgba(124,108,255,.05),transparent 60%)} +body:after{content:'';position:fixed;inset:0;pointer-events:none;z-index:0; + background-image:linear-gradient(rgba(120,160,255,.035) 1px,transparent 1px),linear-gradient(90deg,rgba(120,160,255,.035) 1px,transparent 1px); + background-size:46px 46px;mask-image:radial-gradient(80% 60% at 50% 20%,#000 30%,transparent 100%)} +.wrap{position:relative;z-index:1;max-width:1180px;margin:0 auto;padding:26px 20px 80px} +nav{position:sticky;top:0;z-index:5;background:rgba(10,14,20,.94);border-bottom:1px solid var(--line);border-image:linear-gradient(90deg,transparent,rgba(124,108,255,.5),transparent) 1} +nav .in{max-width:1180px;margin:0 auto;display:flex;gap:18px;align-items:center;padding:13px 20px} +nav a{color:var(--dim);text-decoration:none;font-weight:600;font-size:14px;padding:5px 12px;border-radius:9px} +nav a.on,nav a:hover{color:#fff;background:rgba(124,108,255,.14)} +.brand{font-weight:800;letter-spacing:.5px;background:linear-gradient(90deg,#fff,#7c6cff 60%,#35d0ba);background-clip:text;-webkit-text-fill-color:transparent;font-size:17px} +h1.grad{font-size:30px;font-weight:800;background:linear-gradient(90deg,#fff,#9d8fff);background-clip:text;-webkit-text-fill-color:transparent;margin-bottom:6px} +.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:14px;margin:20px 0} +.card{background:var(--panel);border:1px solid var(--line);border-radius:16px;padding:16px 18px;box-shadow:0 12px 30px rgba(0,0,0,.35),inset 0 1px 0 rgba(255,255,255,.05);transition:transform .15s,box-shadow .15s} +.card:hover{transform:translateY(-2px);box-shadow:0 16px 36px rgba(0,0,0,.45),inset 0 1px 0 rgba(255,255,255,.06)} +.card .n{font-size:26px;font-weight:800;background:linear-gradient(90deg,#fff,#35d0ba);background-clip:text;-webkit-text-fill-color:transparent} +.card .l{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:1px;margin-top:2px} +.panel{background:var(--panel);border:1px solid var(--line);border-radius:16px;padding:20px;margin:16px 0;box-shadow:0 12px 30px rgba(0,0,0,.35),inset 0 1px 0 rgba(255,255,255,.05)} +.panel h2{font-size:16px;margin-bottom:12px;color:#fff} +label{display:block;color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:1px;margin:10px 0 4px} +input[type=text],input[type=number],textarea,select{width:100%;background:rgba(8,12,20,.7);border:1px solid var(--line);border-radius:10px;color:var(--txt);padding:9px 12px;font-size:14px;outline:none;transition:border .15s,box-shadow .15s} +input:focus,textarea:focus,select:focus{border-color:rgba(124,108,255,.6);box-shadow:0 0 0 3px rgba(124,108,255,.16)} +textarea{min-height:180px;font:13px ui-monospace,Menlo,monospace;resize:vertical} +button,.btn{display:inline-block;background:linear-gradient(135deg,#7c6cff,#5b48e8);color:#fff;border:none;border-radius:11px;padding:10px 18px;font-weight:700;font-size:14px;cursor:pointer;text-decoration:none;box-shadow:0 6px 18px rgba(124,108,255,.3);transition:transform .12s,filter .12s} +button:hover,.btn:hover{transform:translateY(-1px);filter:brightness(1.12)} +button:active{transform:translateY(1px)} +button.ghost{background:rgba(124,108,255,.12);box-shadow:none;color:#b7aaff} +button.teal{background:linear-gradient(135deg,#35d0ba,#22a893);box-shadow:0 6px 18px rgba(53,208,186,.25)} +button.bad{background:linear-gradient(135deg,#ff6b6b,#e04848);box-shadow:0 6px 18px rgba(255,107,107,.25)} +.row{display:flex;gap:12px;flex-wrap:wrap;align-items:flex-end} +.row>div{flex:1;min-width:150px} +table{width:100%;border-collapse:collapse;font-size:13px} +th{color:var(--dim);text-align:left;font-size:11px;text-transform:uppercase;letter-spacing:1px;padding:8px 10px;border-bottom:1px solid var(--line)} +td{padding:8px 10px;border-bottom:1px solid rgba(120,160,255,.07)} +tr:hover td{background:rgba(124,108,255,.05)} +.chip{display:inline-block;padding:2px 9px;border-radius:20px;font-size:11px;font-weight:700} +.chip.c-doc{background:rgba(124,108,255,.18);color:#b7aaff}.chip.c-ebook{background:rgba(53,208,186,.16);color:#6fe8d6} +.chip.c-financial{background:rgba(255,180,84,.16);color:#ffcf8f}.chip.c-image{background:rgba(53,130,255,.16);color:#8fc0ff} +.chip.c-archive{background:rgba(160,160,180,.14);color:#c3c9dd}.chip.c-unknown{background:rgba(160,160,180,.14);color:#aab} +.bar{height:7px;border-radius:5px;background:rgba(120,160,255,.12);overflow:hidden;margin:8px 0} +.bar i{display:block;height:100%;background:linear-gradient(90deg,#7c6cff,#35d0ba);transition:width .3s} +.muted{color:var(--dim);font-size:13px} +.plan{border:1px solid var(--line);border-radius:14px;padding:14px 16px;margin:10px 0;background:rgba(12,17,28,.6);transition:transform .12s} +.plan:hover{transform:translateY(-1px)} +.plan .src{font-family:ui-monospace,Menlo,monospace;font-size:12px;color:var(--dim);word-break:break-all} +.plan .arrow{color:var(--acc2);font-size:16px;margin:6px 0} +.plan .dst{font-family:ui-monospace,Menlo,monospace;font-size:13px;color:#cfe3ff;word-break:break-all} +.plan .meta{display:flex;gap:10px;flex-wrap:wrap;align-items:center;margin-top:8px} +.v-APPROVE{color:#6fe8d6}.v-REVIEW{color:#ffcf8f}.v-REJECT{color:#ff9c9c} +.dupe{background:rgba(255,180,84,.1);border:1px solid rgba(255,180,84,.35);border-radius:14px;padding:14px 16px;margin:10px 0} +.toast{position:fixed;bottom:22px;right:22px;background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:12px 18px;box-shadow:0 14px 40px rgba(0,0,0,.5);z-index:9;display:none} +@media (prefers-reduced-motion:reduce){*{animation:none!important;transition:none!important}} +""" + +def shell(body, active="home"): + nav = "" + for k, label in [("home","Dashboard"),("files","Files"),("plan","Plan"),("dupes","Duplicates"),("sessions","Sessions"),("settings","Settings")]: + nav += f"{label}" + return ("" + "LIBRA — File Librarian" + "" + "
" + body + "
" + "
" + "") + +@app.route("/") +def home(): + refresh_counts() + err = f"

Last error

{STATE.get('last_error','')}
" if STATE.get("last_error") else "" + scan_status = "
scan running… refresh page" if STATE["scan_running"] else "" + cls_status = f"
classifying {STATE['classify_done']}/{STATE['classify_total']}…" if STATE["classify_running"] else "" + body = f""" +

File Librarian

Scanner → Extractor → Nimble → Safety → You approve → Reversible move. Nothing moves without your click.

+
+
{STATE.get('files',0)}
Files indexed
+
{STATE.get('classified',0)}
Classified
+
{STATE.get('dupes',0)}
In dup groups
+
{STATE.get('pending',0)}
Approved (dry)
+
{STATE.get('moved',0)}
Moved
+
{err} +

1 · Scan a location

{scan_status} +
+
+
+
+
+
+
+

2 · Hash + Classify (Nimble on nightmare)

{cls_status} +
+
+
+
+
+
+
+
+
+

Pipeline

scan → hash (deterministic, sha256) → preview extract (pdf/zip/text/images) → Nimble structured JSON (think:false) → safety validate → review plan → apply (dry-run default) → immutable op log → one-click undo.

+""" + return Response(shell(body, "home"), mimetype="text/html") + +@app.route("/scan", methods=["POST"]) +def scan_post(): + root = request.form.get("root", "~/Downloads") + if STATE["scan_running"]: + return redirect("/") + threading.Thread(target=scan_thread, args=(root, int(request.form.get("max_files", 2000)), + int(request.form.get("min_size", 0)), int(request.form.get("max_size", 0))), daemon=True).start() + return redirect("/") + +@app.route("/hash", methods=["POST"]) +def hash_post(): + if not STATE["classify_running"]: + threading.Thread(target=hash_thread, args=(int(request.form.get("limit", 500)), + int(request.form.get("max_mb", 64))), daemon=True).start() + return redirect("/") + +@app.route("/classify", methods=["POST"]) +def classify_post(): + if not STATE["classify_running"]: + threading.Thread(target=classify_thread, args=(int(request.form.get("limit", 100)),), daemon=True).start() + return redirect("/") + +@app.route("/files") +def files_page(): + q = request.args.get("q", ""); cls = request.args.get("cls", "") + c = db() + sql = "SELECT * FROM files WHERE 1"; args = [] + if q: sql += " AND (path LIKE ? OR name LIKE ?)"; args += [f"%{q}%", f"%{q}%"] + if cls: sql += " AND file_class=?"; args.append(cls) + sql += " ORDER BY id DESC LIMIT 400" + rows = c.execute(sql, args).fetchall() + c.close() + trs = "" + for r in rows: + cls_chip = f"{r['file_class'] or '—'}" if r["file_class"] else "unclassified" + subj = r["subject"] or "" + conf = f"{int((r['confidence'] or 0)*100)}%" if r["confidence"] is not None else "" + dup = " ◆dup" if r["sha256"] and r["sha256"] not in ("TOOBIG","UNREADABLE") else "" + pv = (r["preview"] or "")[:110] + trs += (f"{cls_chip} {subj} {conf}{dup}" + f"
{r['path']}
{pv}
" + f"{r['status']}") + body = f""" +

Indexed Files

+
+
+
+
+
+
{trs}
ClassPath + previewStatus
+""" + return Response(shell(body, "files"), mimetype="text/html") + +@app.route("/plan") +def plan_page(): + tx = load_taxonomy() + min_conf = float(request.args.get("min_conf", 0.6)) + plans = build_plan(tx, min_conf) + plans.sort(key=lambda p: (-p["conf"], p["verdict"] != "APPROVE")) + cards = "" + approved_default = 0 + for p in plans[:600]: + checked = "checked" if p["verdict"] == "APPROVE" else "" + if p["verdict"] == "APPROVE": approved_default += 1 + dupnote = "DUPLICATE GROUP" if p["dup"] else "" + cards += (f"
" + f"
{p['path']}
↓
{p['dst']}
" + f"
{p['cls']}{p['subj']} · {p['imp']}" + f"conf {int(p['conf']*100)}%" + f"{p['verdict']}" + + (f"({p['why']})" if p["why"] else "") + dupnote + + f"" + f"
") + body = f""" +

Organization Plan

+
+
+
+
+

{len(plans)} proposals · {approved_default} auto-approved at this threshold. Uncheck anything you're unsure of — nothing moves without your click.

+
+
+
{cards}
+

Progress

+ +""" + return Response(shell(body, "plan"), mimetype="text/html") + +@app.route("/api/apply", methods=["POST"]) +def api_apply(): + d = request.get_json(force=True) + ids = [int(x) for x in d.get("ids", [])] + done, skipped, session = apply_moves(ids, dry_run=bool(d.get("dry_run", True))) + return {"done": done, "skipped": skipped, "session": session} + +@app.route("/dupes") +def dupes_page(): + c = db() + groups = c.execute("""SELECT sha256, COUNT(*) n, SUM(size) b FROM files + WHERE sha256 IS NOT NULL AND sha256 NOT IN ('TOOBIG','UNREADABLE') + GROUP BY sha256 HAVING COUNT(*)>1 ORDER BY b DESC LIMIT 100""").fetchall() + c.close() + out = "" + reclaim = 0 + for g in groups: + c = db() + members = c.execute("SELECT * FROM files WHERE sha256=?", (g["sha256"],)).fetchall() + c.close() + reclaim += g["b"] * (g["n"] - 1) + rows = "" + for m in members: + rows += (f"
" + f"{m['path']}" + f"
") + out += f"
{g['n']} copies · {g['b']/1e6:.1f} MB each{rows}
" + body = f""" +

Duplicates

+

sha256-verified only — never guessed. Keepers stay; the rest go to ~/libra/dupe_hold/ (reversible). {reclaim/1e9:.2f} GB reclaimable.

+
{out or "

No duplicate groups yet — run Hash on the dashboard.

"}
+""" + return Response(shell(body, "dupes"), mimetype="text/html") + +@app.route("/api/keep", methods=["POST"]) +def api_keep(): + d = request.get_json(force=True) + n = delete_duplicates(int(d["keep"])) + return {"held": n} + +@app.route("/sessions") +def sessions_page(): + c = db() + sess = c.execute("SELECT * FROM sessions ORDER BY ts DESC LIMIT 50").fetchall() + ops = c.execute("SELECT * FROM operations ORDER BY ts DESC LIMIT 200").fetchall() + c.close() + sr = "" + for s in sess: + undo_btn = "" if s["undone"] or s["note"] == "duplicates to hold" else f"" + sr += (f"
{s['id']}" + f"{time.strftime('%b %d %H:%M', time.localtime(s['ts']))} · {s['n']} ops · {s['note']}" + + (" · UNDONE" if s["undone"] else "") + "" + f"{undo_btn}
") + orows = "" + for o in ops: + orows += (f"{time.strftime('%m-%d %H:%M', time.localtime(o['ts']))}{o['op']}" + f"{o['src']}" + f"{o['dst']}" + f"{o['session']}" + (" ↩" if o["undone"] else "") + "") + body = f""" +

Sessions · Operation Log

+

Every move is logged with source, destination and sha256. Undo restores files exactly.

+

Sessions

{sr or 'no sessions yet'}
+

Operation log

{orows}
WhenOpSourceDestinationSession
+""" + return Response(shell(body, "sessions"), mimetype="text/html") + +@app.route("/api/undo", methods=["POST"]) +def api_undo(): + d = request.get_json(force=True) + n = undo_session(d["session"]) + return {"undone": n} + +@app.route("/settings") +def settings_page(): + tx = open(CFG_PATH).read() + body = f""" +

Settings

+

Taxonomy (destination map)

+
+ + +
+
+

Engine

+

Model: {MODEL} @ {OLLAMA} (Nimble decision lane, think:false) · DB: {DB_PATH}

+

Safety gates: no system paths · no symlinks · no overwrite (auto-suffix) · confidence gate on plan · every move logged + reversible · dupes go to hold, never deleted.

+""" + return Response(shell(body, "settings"), mimetype="text/html") + +@app.route("/settings/save", methods=["POST"]) +def settings_save(): + open(CFG_PATH, "w").write(request.form.get("tax", "")) + return redirect("/settings") + +if __name__ == "__main__": + init_db() + refresh_counts() + app.run(host="127.0.0.1", port=5731, debug=False) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..15639a1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +flask +pypdf +Pillow diff --git a/taxonomy.yaml b/taxonomy.yaml new file mode 100644 index 0000000..a807fb1 --- /dev/null +++ b/taxonomy.yaml @@ -0,0 +1,31 @@ +# Libra taxonomy — destination map (editable here) +# mapping: '+' -> destination (relative to organized root) +organized_root: ~/libra-organized +map: + document+finance: Documents/Financial + document+work: Documents/Work + document+personal: Documents/Personal + document+unknown: Documents/Misc + ebook+reference: Knowledge/Books + ebook+programming: Knowledge/Books + ebook+AI: Knowledge/Books + ebook+linux: Knowledge/Books + ebook+cybersecurity: Knowledge/Books + ebook+unknown: Knowledge/Books + ebook+personal: Knowledge/Books + image+photography: Media/Photos + image+personal: Media/Photos + image+unknown: Media/Photos + video+unknown: Media/Videos + audio+unknown: Media/Audio + software+unknown: Software/Installers + software+programming: Software/Source + source_code+programming: Software/Source + source_code+AI: Software/Source + archive+unknown: Data/Archives + dataset+unknown: Data/Datasets + financial+finance: Documents/Financial + personal+personal: Documents/Personal + temporary+unknown: Unsorted + unknown+unknown: Unsorted +default: Unsorted