LIBRA v1: scanner/extractor/Nimble classifier/safety gate/approval UI/op log/undo/dupe hold
This commit is contained in:
drjones
712
app.py
Normal file
712
app.py
Normal file
@@ -0,0 +1,712 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LIBRA — the Scales. Nimble-classified file organizer with approval-gated, reversible moves.
|
||||
Local only: Flask on Mac, Ollama Nimble decision lane on nightmare .29."""
|
||||
import os, re, json, time, uuid, hashlib, sqlite3, threading, shutil, traceback, fnmatch
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from flask import Flask, request, redirect, Response
|
||||
|
||||
APP_DIR = os.path.expanduser("~/libra")
|
||||
DB_PATH = os.path.join(APP_DIR, "libra.db")
|
||||
CFG_PATH = os.path.join(APP_DIR, "taxonomy.yaml")
|
||||
|
||||
OLLAMA = "http://10.30.20.29:11434"
|
||||
MODEL = "nimble:latest"
|
||||
HOME = os.path.expanduser("~")
|
||||
|
||||
DEFAULT_TAXONOMY = """# Libra taxonomy — destination map (editable here)
|
||||
# mapping: '<file_class>+<subject>' -> destination (relative to organized root)
|
||||
organized_root: ~/libra-organized
|
||||
map:
|
||||
document+finance: Documents/Financial
|
||||
document+work: Documents/Work
|
||||
document+personal: Documents/Personal
|
||||
document+unknown: Documents/Misc
|
||||
ebook+reference: Knowledge/Books
|
||||
ebook+programming: Knowledge/Books
|
||||
ebook+AI: Knowledge/Books
|
||||
ebook+linux: Knowledge/Books
|
||||
ebook+cybersecurity: Knowledge/Books
|
||||
ebook+unknown: Knowledge/Books
|
||||
ebook+personal: Knowledge/Books
|
||||
image+photography: Media/Photos
|
||||
image+personal: Media/Photos
|
||||
image+unknown: Media/Photos
|
||||
video+unknown: Media/Videos
|
||||
audio+unknown: Media/Audio
|
||||
software+unknown: Software/Installers
|
||||
software+programming: Software/Source
|
||||
source_code+programming: Software/Source
|
||||
source_code+AI: Software/Source
|
||||
archive+unknown: Data/Archives
|
||||
dataset+unknown: Data/Datasets
|
||||
financial+finance: Documents/Financial
|
||||
personal+personal: Documents/Personal
|
||||
temporary+unknown: Unsorted
|
||||
unknown+unknown: Unsorted
|
||||
default: Unsorted
|
||||
"""
|
||||
|
||||
if not os.path.exists(CFG_PATH):
|
||||
Path(APP_DIR).mkdir(parents=True, exist_ok=True)
|
||||
open(CFG_PATH, "w").write(DEFAULT_TAXONOMY)
|
||||
|
||||
app = Flask(__name__)
|
||||
LOCK = threading.Lock()
|
||||
STATE = {"scan_running": False, "scan_done": 0, "scan_total": 0, "classify_running": False,
|
||||
"classify_done": 0, "classify_total": 0, "last_error": ""}
|
||||
|
||||
# ---------- DB ----------
|
||||
def db():
|
||||
c = sqlite3.connect(DB_PATH, timeout=30)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
def init_db():
|
||||
c = db()
|
||||
c.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS files(
|
||||
id INTEGER PRIMARY KEY, path TEXT UNIQUE, name TEXT, ext TEXT, size INTEGER, mtime REAL,
|
||||
sha256 TEXT, preview TEXT, file_class TEXT, subject TEXT, importance TEXT, action TEXT,
|
||||
confidence REAL, rename TEXT, status TEXT DEFAULT 'scanned', reason TEXT);
|
||||
CREATE TABLE IF NOT EXISTS operations(
|
||||
id INTEGER PRIMARY KEY, ts REAL, session TEXT, file_id INTEGER, op TEXT,
|
||||
src TEXT, dst TEXT, sha256 TEXT, undone INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS sessions(
|
||||
id TEXT PRIMARY KEY, ts REAL, n INTEGER, undone INTEGER DEFAULT 0, note TEXT);
|
||||
""")
|
||||
c.commit(); c.close()
|
||||
|
||||
# ---------- SCANNER ----------
|
||||
SKIP_DIRS = {".Trashes", ".Spotlight-V100", ".TemporaryItems", ".fseventsd", "System Volume Information",
|
||||
"$RECYCLE.BIN", "node_modules", "__pycache__", ".git", "Library", ".Trash"}
|
||||
JUNK = (".DS_Store",)
|
||||
|
||||
def scan(root, max_files=5000, min_size=0, max_size=0):
|
||||
root = os.path.abspath(os.path.expanduser(root))
|
||||
c = db()
|
||||
n = 0
|
||||
for r, dirs, fs in os.walk(root):
|
||||
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith("._")]
|
||||
for f in fs:
|
||||
if f in JUNK or f.startswith("._"):
|
||||
continue
|
||||
p = os.path.join(r, f)
|
||||
try:
|
||||
st = os.lstat(p)
|
||||
except OSError:
|
||||
continue
|
||||
if st.st_size < min_size or (max_size and st.st_size > max_size):
|
||||
continue
|
||||
try:
|
||||
c.execute("INSERT OR IGNORE INTO files(path,name,ext,size,mtime,status) VALUES(?,?,?,?,?, 'scanned')",
|
||||
(p, f, os.path.splitext(f)[1].lower(), st.st_size, st.st_mtime))
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
n += 1
|
||||
if n >= max_files:
|
||||
break
|
||||
if n >= max_files:
|
||||
break
|
||||
c.commit(); c.close()
|
||||
return n
|
||||
|
||||
def scan_thread(root, max_files, min_size, max_size):
|
||||
try:
|
||||
with LOCK:
|
||||
STATE["scan_running"] = True
|
||||
scan(root, max_files, min_size, max_size)
|
||||
except Exception:
|
||||
STATE["last_error"] = traceback.format_exc()[-800:]
|
||||
finally:
|
||||
with LOCK:
|
||||
STATE["scan_running"] = False
|
||||
refresh_counts()
|
||||
|
||||
def refresh_counts():
|
||||
c = db()
|
||||
for k, q in [("files", "SELECT COUNT(*) FROM files"),
|
||||
("scanned", "SELECT COUNT(*) FROM files WHERE file_class IS NULL"),
|
||||
("classified", "SELECT COUNT(*) FROM files WHERE file_class IS NOT NULL"),
|
||||
("dupes", "SELECT COUNT(*) FROM files WHERE sha256 IS NOT NULL AND sha256 IN (SELECT sha256 FROM files WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(*)>1)"),
|
||||
("pending", "SELECT COUNT(*) FROM files WHERE status='approved'"),
|
||||
("moved", "SELECT COUNT(*) FROM files WHERE status='moved'")]:
|
||||
STATE[k] = c.execute(q).fetchone()[0]
|
||||
c.close()
|
||||
|
||||
# ---------- EXTRACTOR ----------
|
||||
def extract_preview(p, ext, size):
|
||||
prev = ""
|
||||
try:
|
||||
if ext == ".pdf" and size < 60_000_000:
|
||||
from pypdf import PdfReader
|
||||
rd = PdfReader(p)
|
||||
txt = ""
|
||||
for pg in rd.pages[:2]:
|
||||
txt += (pg.extract_text() or "") + " "
|
||||
prev = re.sub(r"\s+", " ", txt)[:600]
|
||||
elif ext in (".txt", ".md", ".csv", ".json", ".yaml", ".yml", ".log", ".py", ".sh", ".html", ".cfg", ".ini") and size < 5_000_000:
|
||||
prev = open(p, "r", errors="replace").read(600)
|
||||
elif ext in (".zip",) and size < 500_000_000:
|
||||
import zipfile
|
||||
with zipfile.ZipFile(p) as z:
|
||||
names = z.namelist()[:20]
|
||||
prev = "zip listing: " + ", ".join(names)
|
||||
elif ext in (".jpg", ".jpeg", ".png", ".heic", ".gif", ".webp"):
|
||||
try:
|
||||
from PIL import Image
|
||||
im = Image.open(p)
|
||||
prev = f"image {im.size[0]}x{im.size[1]}"
|
||||
except Exception:
|
||||
prev = "image file"
|
||||
except Exception:
|
||||
prev = ""
|
||||
return prev
|
||||
|
||||
def hash_files(limit=500, max_mb=64):
|
||||
c = db()
|
||||
rows = c.execute("SELECT id,path,size FROM files WHERE sha256 IS NULL LIMIT ?", (limit,)).fetchall()
|
||||
for row in rows:
|
||||
if row["size"] > max_mb * 1_000_000:
|
||||
c.execute("UPDATE files SET sha256='TOOBIG' WHERE id=?", (row["id"],)); continue
|
||||
h = hashlib.sha256()
|
||||
try:
|
||||
with open(row["path"], "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(4 << 20), b""):
|
||||
h.update(chunk)
|
||||
c.execute("UPDATE files SET sha256=? WHERE id=?", (h.hexdigest(), row["id"]))
|
||||
except OSError:
|
||||
c.execute("UPDATE files SET sha256='UNREADABLE' WHERE id=?", (row["id"],))
|
||||
c.commit(); n = len(rows); c.close(); return n
|
||||
|
||||
def hash_thread(limit, max_mb):
|
||||
try:
|
||||
with LOCK: STATE["classify_running"] = True
|
||||
while hash_files(limit, max_mb):
|
||||
time.sleep(0.1)
|
||||
except Exception:
|
||||
STATE["last_error"] = traceback.format_exc()[-800:]
|
||||
finally:
|
||||
with LOCK: STATE["classify_running"] = False
|
||||
refresh_counts()
|
||||
|
||||
# ---------- CLASSIFIER (Nimble) ----------
|
||||
ENUMS = {
|
||||
"file_class": ["document","image","video","audio","software","source_code","archive","dataset","ebook","financial","personal","temporary","unknown"],
|
||||
"subject": ["work","finance","programming","AI","linux","cybersecurity","photography","personal","entertainment","reference","unknown"],
|
||||
"importance": ["disposable","low","normal","important","critical"],
|
||||
"action": ["keep","organize","archive","quarantine","review"],
|
||||
}
|
||||
def classify_one(row):
|
||||
prompt = ("Classify this file. Reply ONLY minified JSON with keys file_class, subject, importance, action, confidence, rename. No other text.\n"
|
||||
f"File: {row['name']} ({row['ext'] or 'no ext'}, {row['size']} bytes)\n"
|
||||
+ (f"Content preview: {row['preview'][:400]}\n" if row['preview'] else "")
|
||||
+ "file_class in " + str(ENUMS["file_class"]) + "; subject in " + str(ENUMS["subject"])
|
||||
+ "; importance in " + str(ENUMS["importance"]) + "; action in " + str(ENUMS["action"])
|
||||
+ "; confidence 0.0-1.0; rename = clean proposed filename or same")
|
||||
import urllib.request
|
||||
body = json.dumps({"model": MODEL, "prompt": prompt, "stream": False, "think": False,
|
||||
"options": {"temperature": 0.1, "num_predict": 400}}).encode()
|
||||
req = urllib.request.Request(OLLAMA + "/api/generate", data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
d = json.loads(r.read())
|
||||
txt = d.get("response", "")
|
||||
m = re.search(r"\{.*\}", txt, re.S)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
j = json.loads(m.group(0))
|
||||
except Exception:
|
||||
return None
|
||||
out = {}
|
||||
for k, vals in ENUMS.items():
|
||||
v = str(j.get(k, "unknown")).strip().lower()
|
||||
if v not in vals: v = "unknown" if k != "importance" else "normal"
|
||||
out[k] = v
|
||||
try: conf = float(j.get("confidence", 0.5))
|
||||
except Exception: conf = 0.5
|
||||
out["confidence"] = max(0.0, min(1.0, conf if conf <= 1.0 else conf / 10.0))
|
||||
rn = str(j.get("rename", "")).strip()
|
||||
out["rename"] = re.sub(r"[^A-Za-z0-9 ._-]", "", rn)[:120] or row["name"]
|
||||
return out
|
||||
|
||||
def classify_thread(limit):
|
||||
try:
|
||||
with LOCK: STATE["classify_running"] = True
|
||||
c = db()
|
||||
rows = c.execute("SELECT * FROM files WHERE file_class IS NULL LIMIT ?", (limit,)).fetchall()
|
||||
STATE["classify_total"] = len(rows); STATE["classify_done"] = 0
|
||||
for row in rows:
|
||||
res = classify_one(row)
|
||||
if res:
|
||||
if row["preview"] is None:
|
||||
prev = extract_preview(row["path"], row["ext"], row["size"])
|
||||
c.execute("UPDATE files SET preview=? WHERE id=?", (prev, row["id"]))
|
||||
c.execute("UPDATE files SET file_class=?,subject=?,importance=?,action=?,confidence=?,rename=?,status='classified' WHERE id=?",
|
||||
(res["file_class"], res["subject"], res["importance"], res["action"], res["confidence"], res["rename"], row["id"]))
|
||||
STATE["classify_done"] += 1
|
||||
c.commit(); c.close()
|
||||
except Exception:
|
||||
STATE["last_error"] = traceback.format_exc()[-800:]
|
||||
finally:
|
||||
with LOCK: STATE["classify_running"] = False
|
||||
refresh_counts()
|
||||
|
||||
# ---------- TAXONOMY + SAFETY ----------
|
||||
def load_taxonomy():
|
||||
tx = {"organized_root": os.path.expanduser("~/libra-organized"), "map": {}, "default": "Unsorted"}
|
||||
cur = None
|
||||
for line in open(CFG_PATH):
|
||||
line = line.rstrip()
|
||||
if not line or line.lstrip().startswith("#"): continue
|
||||
if line.startswith("organized_root:"):
|
||||
tx["organized_root"] = os.path.expanduser(line.split(":", 1)[1].strip())
|
||||
elif line.startswith("default:"):
|
||||
tx["default"] = line.split(":", 1)[1].strip()
|
||||
elif line.startswith("map:"):
|
||||
cur = "map"
|
||||
elif cur == "map" and line.startswith(" ") and ":" in line:
|
||||
k, v = line.strip().split(":", 1)
|
||||
tx["map"][k.strip()] = v.strip()
|
||||
return tx
|
||||
|
||||
def propose_destination(row, tx):
|
||||
key = f"{row['file_class']}+{row['subject']}"
|
||||
sub = tx["map"].get(key, tx["map"].get(f"{row['file_class']}+unknown", tx["default"]))
|
||||
date = time.strftime("%Y/%m", time.localtime(row["mtime"] or time.time()))
|
||||
return os.path.join(tx["organized_root"], sub, date)
|
||||
|
||||
def validate(src, dst, confidence, min_conf, tx):
|
||||
if confidence < min_conf: return "REVIEW", "confidence below threshold"
|
||||
if not src.startswith(tx.get("scan_root", "/")): return "REJECT", "outside scan root"
|
||||
for sysd in ("/System", "/usr", "/etc", "/bin", "/sbin", "/var", "/private", "/opt"):
|
||||
if src.startswith(sysd + "/"): return "REJECT", "system path"
|
||||
if os.path.islink(src): return "REJECT", "symlink"
|
||||
if os.path.exists(dst): return "REVIEW", "destination exists"
|
||||
if os.path.dirname(src) == dst: return "REJECT", "already there"
|
||||
return "APPROVE", ""
|
||||
|
||||
def build_plan(tx, min_conf):
|
||||
c = db()
|
||||
rows = c.execute("SELECT * FROM files WHERE status='classified'").fetchall()
|
||||
c.close()
|
||||
dupset = set()
|
||||
c = db()
|
||||
for (s,) in c.execute("SELECT sha256 FROM files WHERE sha256 IS NOT NULL AND sha256 NOT IN ('TOOBIG','UNREADABLE') GROUP BY sha256 HAVING COUNT(*)>1"):
|
||||
dupset.add(s)
|
||||
c.close()
|
||||
plans = []
|
||||
for row in rows:
|
||||
if row["action"] not in ("organize", "archive"): continue
|
||||
dst_dir = propose_destination(row, tx)
|
||||
name = row["rename"] if row["rename"] else row["name"]
|
||||
base, ext = os.path.splitext(name)
|
||||
if ext.lower() != row["ext"] and row["ext"]:
|
||||
name = base + row["ext"]
|
||||
dst = os.path.join(dst_dir, name)
|
||||
verdict, why = validate(row["path"], dst, row["confidence"], min_conf, tx)
|
||||
is_dup = row["sha256"] in dupset
|
||||
plans.append({"id": row["id"], "path": row["path"], "name": row["name"], "dst": dst,
|
||||
"cls": row["file_class"], "subj": row["subject"], "imp": row["importance"],
|
||||
"conf": row["confidence"], "verdict": verdict, "why": why, "dup": is_dup,
|
||||
"size": row["size"]})
|
||||
return plans
|
||||
|
||||
# ---------- ORGANIZER (apply / undo) ----------
|
||||
def apply_moves(ids, dry_run=True):
|
||||
tx = load_taxonomy()
|
||||
c = db()
|
||||
session = uuid.uuid4().hex[:12]
|
||||
done, skipped = 0, 0
|
||||
for fid in ids:
|
||||
row = c.execute("SELECT * FROM files WHERE id=?", (fid,)).fetchone()
|
||||
if not row or row["status"] not in ("classified", "approved"): skipped += 1; continue
|
||||
dst_dir = propose_destination(row, tx)
|
||||
name = row["rename"] or row["name"]
|
||||
base, ext = os.path.splitext(name)
|
||||
if ext.lower() != row["ext"] and row["ext"]:
|
||||
name = base + row["ext"]
|
||||
dst = os.path.join(dst_dir, name)
|
||||
i = 1
|
||||
while os.path.exists(dst):
|
||||
dst = os.path.join(dst_dir, f"{base}({i}){ext}"); i += 1
|
||||
verdict, why = validate(row["path"], dst, row["confidence"], 0.0, tx)
|
||||
if verdict == "REJECT":
|
||||
skipped += 1; continue
|
||||
if dry_run:
|
||||
c.execute("UPDATE files SET status='approved' WHERE id=?", (fid,))
|
||||
done += 1; continue
|
||||
try:
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
shutil.move(row["path"], dst)
|
||||
c.execute("INSERT INTO operations(ts,session,file_id,op,src,dst,sha256) VALUES(?,?,?,?,?,?,?)",
|
||||
(time.time(), session, fid, "move", row["path"], dst, row["sha256"] or ""))
|
||||
c.execute("UPDATE files SET path=?,name=?,status='moved' WHERE id=?", (dst, os.path.basename(dst), fid))
|
||||
done += 1
|
||||
except OSError as e:
|
||||
skipped += 1
|
||||
STATE["last_error"] = f"{e}"
|
||||
if not dry_run:
|
||||
c.execute("INSERT INTO sessions(id,ts,n,note) VALUES(?,?,?,?)", (session, time.time(), done, "apply"))
|
||||
c.commit(); c.close(); refresh_counts()
|
||||
return done, skipped, session
|
||||
|
||||
def undo_session(session):
|
||||
c = db()
|
||||
ops = c.execute("SELECT * FROM operations WHERE session=? AND undone=0 ORDER BY id DESC", (session,)).fetchall()
|
||||
n = 0
|
||||
for op in ops:
|
||||
try:
|
||||
if os.path.exists(op["dst"]):
|
||||
os.makedirs(os.path.dirname(op["src"]), exist_ok=True)
|
||||
shutil.move(op["dst"], op["src"])
|
||||
c.execute("UPDATE files SET path=?,name=?,status='classified' WHERE id=?", (op["src"], os.path.basename(op["src"]), op["file_id"]))
|
||||
c.execute("UPDATE operations SET undone=1 WHERE id=?", (op["id"],))
|
||||
n += 1
|
||||
except OSError as e:
|
||||
STATE["last_error"] = str(e)
|
||||
c.execute("UPDATE sessions SET undone=1 WHERE id=?", (session,))
|
||||
c.commit(); c.close(); refresh_counts()
|
||||
return n
|
||||
|
||||
def delete_duplicates(keep_id):
|
||||
c = db()
|
||||
row = c.execute("SELECT sha256 FROM files WHERE id=?", (keep_id,)).fetchone()
|
||||
if not row or not row["sha256"]: c.close(); return 0
|
||||
session = uuid.uuid4().hex[:12]
|
||||
n = 0
|
||||
for d in c.execute("SELECT * FROM files WHERE sha256=? AND id!=? AND status!='moved'", (row["sha256"], keep_id)).fetchall():
|
||||
try:
|
||||
hold = os.path.join(APP_DIR, "dupe_hold", session)
|
||||
os.makedirs(hold, exist_ok=True)
|
||||
shutil.move(d["path"], os.path.join(hold, f"{n:04d}__" + d["name"]))
|
||||
c.execute("INSERT INTO operations(ts,session,file_id,op,src,dst,sha256) VALUES(?,?,?,?,?,?,?)",
|
||||
(time.time(), session, d["id"], "dupe_hold", d["path"], os.path.join(hold, d["name"]), d["sha256"]))
|
||||
c.execute("UPDATE files SET status='held' WHERE id=?", (d["id"],))
|
||||
n += 1
|
||||
except OSError:
|
||||
pass
|
||||
c.execute("INSERT INTO sessions(id,ts,n,note) VALUES(?,?,?,?)", (session, time.time(), n, "duplicates to hold"))
|
||||
c.commit(); c.close(); refresh_counts()
|
||||
return n
|
||||
|
||||
# ---------- UI ----------
|
||||
BASE_CSS = """
|
||||
:root{--bg:#0a0e14;--panel:rgba(20,26,38,.78);--line:rgba(120,160,255,.14);--txt:#dbe4f5;--dim:#8194b8;--acc:#7c6cff;--acc2:#35d0ba;--warn:#ffb454;--bad:#ff6b6b}
|
||||
*{box-sizing:border-box;margin:0}
|
||||
body{color:var(--txt);font:15px/1.55 -apple-system,'SF Pro Text',Helvetica,sans-serif;min-height:100vh;
|
||||
background:linear-gradient(180deg,#0a0e14 0%,#0d1320 60%,#0a0e14 100%) fixed;
|
||||
position:relative}
|
||||
body:before{content:'';position:fixed;inset:0;pointer-events:none;z-index:0;
|
||||
background:radial-gradient(900px 500px at 15% -10%,rgba(124,108,255,.08),transparent 60%),
|
||||
radial-gradient(800px 500px at 90% 20%,rgba(53,208,186,.06),transparent 60%),
|
||||
radial-gradient(700px 400px at 50% 110%,rgba(124,108,255,.05),transparent 60%)}
|
||||
body:after{content:'';position:fixed;inset:0;pointer-events:none;z-index:0;
|
||||
background-image:linear-gradient(rgba(120,160,255,.035) 1px,transparent 1px),linear-gradient(90deg,rgba(120,160,255,.035) 1px,transparent 1px);
|
||||
background-size:46px 46px;mask-image:radial-gradient(80% 60% at 50% 20%,#000 30%,transparent 100%)}
|
||||
.wrap{position:relative;z-index:1;max-width:1180px;margin:0 auto;padding:26px 20px 80px}
|
||||
nav{position:sticky;top:0;z-index:5;background:rgba(10,14,20,.94);border-bottom:1px solid var(--line);border-image:linear-gradient(90deg,transparent,rgba(124,108,255,.5),transparent) 1}
|
||||
nav .in{max-width:1180px;margin:0 auto;display:flex;gap:18px;align-items:center;padding:13px 20px}
|
||||
nav a{color:var(--dim);text-decoration:none;font-weight:600;font-size:14px;padding:5px 12px;border-radius:9px}
|
||||
nav a.on,nav a:hover{color:#fff;background:rgba(124,108,255,.14)}
|
||||
.brand{font-weight:800;letter-spacing:.5px;background:linear-gradient(90deg,#fff,#7c6cff 60%,#35d0ba);background-clip:text;-webkit-text-fill-color:transparent;font-size:17px}
|
||||
h1.grad{font-size:30px;font-weight:800;background:linear-gradient(90deg,#fff,#9d8fff);background-clip:text;-webkit-text-fill-color:transparent;margin-bottom:6px}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:14px;margin:20px 0}
|
||||
.card{background:var(--panel);border:1px solid var(--line);border-radius:16px;padding:16px 18px;box-shadow:0 12px 30px rgba(0,0,0,.35),inset 0 1px 0 rgba(255,255,255,.05);transition:transform .15s,box-shadow .15s}
|
||||
.card:hover{transform:translateY(-2px);box-shadow:0 16px 36px rgba(0,0,0,.45),inset 0 1px 0 rgba(255,255,255,.06)}
|
||||
.card .n{font-size:26px;font-weight:800;background:linear-gradient(90deg,#fff,#35d0ba);background-clip:text;-webkit-text-fill-color:transparent}
|
||||
.card .l{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:1px;margin-top:2px}
|
||||
.panel{background:var(--panel);border:1px solid var(--line);border-radius:16px;padding:20px;margin:16px 0;box-shadow:0 12px 30px rgba(0,0,0,.35),inset 0 1px 0 rgba(255,255,255,.05)}
|
||||
.panel h2{font-size:16px;margin-bottom:12px;color:#fff}
|
||||
label{display:block;color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:1px;margin:10px 0 4px}
|
||||
input[type=text],input[type=number],textarea,select{width:100%;background:rgba(8,12,20,.7);border:1px solid var(--line);border-radius:10px;color:var(--txt);padding:9px 12px;font-size:14px;outline:none;transition:border .15s,box-shadow .15s}
|
||||
input:focus,textarea:focus,select:focus{border-color:rgba(124,108,255,.6);box-shadow:0 0 0 3px rgba(124,108,255,.16)}
|
||||
textarea{min-height:180px;font:13px ui-monospace,Menlo,monospace;resize:vertical}
|
||||
button,.btn{display:inline-block;background:linear-gradient(135deg,#7c6cff,#5b48e8);color:#fff;border:none;border-radius:11px;padding:10px 18px;font-weight:700;font-size:14px;cursor:pointer;text-decoration:none;box-shadow:0 6px 18px rgba(124,108,255,.3);transition:transform .12s,filter .12s}
|
||||
button:hover,.btn:hover{transform:translateY(-1px);filter:brightness(1.12)}
|
||||
button:active{transform:translateY(1px)}
|
||||
button.ghost{background:rgba(124,108,255,.12);box-shadow:none;color:#b7aaff}
|
||||
button.teal{background:linear-gradient(135deg,#35d0ba,#22a893);box-shadow:0 6px 18px rgba(53,208,186,.25)}
|
||||
button.bad{background:linear-gradient(135deg,#ff6b6b,#e04848);box-shadow:0 6px 18px rgba(255,107,107,.25)}
|
||||
.row{display:flex;gap:12px;flex-wrap:wrap;align-items:flex-end}
|
||||
.row>div{flex:1;min-width:150px}
|
||||
table{width:100%;border-collapse:collapse;font-size:13px}
|
||||
th{color:var(--dim);text-align:left;font-size:11px;text-transform:uppercase;letter-spacing:1px;padding:8px 10px;border-bottom:1px solid var(--line)}
|
||||
td{padding:8px 10px;border-bottom:1px solid rgba(120,160,255,.07)}
|
||||
tr:hover td{background:rgba(124,108,255,.05)}
|
||||
.chip{display:inline-block;padding:2px 9px;border-radius:20px;font-size:11px;font-weight:700}
|
||||
.chip.c-doc{background:rgba(124,108,255,.18);color:#b7aaff}.chip.c-ebook{background:rgba(53,208,186,.16);color:#6fe8d6}
|
||||
.chip.c-financial{background:rgba(255,180,84,.16);color:#ffcf8f}.chip.c-image{background:rgba(53,130,255,.16);color:#8fc0ff}
|
||||
.chip.c-archive{background:rgba(160,160,180,.14);color:#c3c9dd}.chip.c-unknown{background:rgba(160,160,180,.14);color:#aab}
|
||||
.bar{height:7px;border-radius:5px;background:rgba(120,160,255,.12);overflow:hidden;margin:8px 0}
|
||||
.bar i{display:block;height:100%;background:linear-gradient(90deg,#7c6cff,#35d0ba);transition:width .3s}
|
||||
.muted{color:var(--dim);font-size:13px}
|
||||
.plan{border:1px solid var(--line);border-radius:14px;padding:14px 16px;margin:10px 0;background:rgba(12,17,28,.6);transition:transform .12s}
|
||||
.plan:hover{transform:translateY(-1px)}
|
||||
.plan .src{font-family:ui-monospace,Menlo,monospace;font-size:12px;color:var(--dim);word-break:break-all}
|
||||
.plan .arrow{color:var(--acc2);font-size:16px;margin:6px 0}
|
||||
.plan .dst{font-family:ui-monospace,Menlo,monospace;font-size:13px;color:#cfe3ff;word-break:break-all}
|
||||
.plan .meta{display:flex;gap:10px;flex-wrap:wrap;align-items:center;margin-top:8px}
|
||||
.v-APPROVE{color:#6fe8d6}.v-REVIEW{color:#ffcf8f}.v-REJECT{color:#ff9c9c}
|
||||
.dupe{background:rgba(255,180,84,.1);border:1px solid rgba(255,180,84,.35);border-radius:14px;padding:14px 16px;margin:10px 0}
|
||||
.toast{position:fixed;bottom:22px;right:22px;background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:12px 18px;box-shadow:0 14px 40px rgba(0,0,0,.5);z-index:9;display:none}
|
||||
@media (prefers-reduced-motion:reduce){*{animation:none!important;transition:none!important}}
|
||||
"""
|
||||
|
||||
def shell(body, active="home"):
|
||||
nav = ""
|
||||
for k, label in [("home","Dashboard"),("files","Files"),("plan","Plan"),("dupes","Duplicates"),("sessions","Sessions"),("settings","Settings")]:
|
||||
nav += f"<a class='{'on' if k==active else ''}' href='/{k}'>{label}</a>"
|
||||
return ("<!doctype html><html><head><meta charset='utf-8'><meta name='viewport' content='width=device-width,initial-scale=1'>"
|
||||
"<title>LIBRA — File Librarian</title><style>" + BASE_CSS + "</style></head><body>"
|
||||
"<nav><div class='in'><span class='brand'>♎ LIBRA</span>" + nav + "</div></nav>"
|
||||
"<div class='wrap'>" + body + "</div>"
|
||||
"<div class='toast' id='toast'></div>"
|
||||
"<script>window.LIBRA=" + json.dumps(STATE) + ";</script></body></html>")
|
||||
|
||||
@app.route("/")
|
||||
def home():
|
||||
refresh_counts()
|
||||
err = f"<div class='panel' style='border-color:rgba(255,107,107,.4)'><h2 style='color:var(--bad)'>Last error</h2><pre style='white-space:pre-wrap;font-size:11px'>{STATE.get('last_error','')}</pre></div>" if STATE.get("last_error") else ""
|
||||
scan_status = "<div class='bar'><i style='width:100%'></i></div><span class='muted'>scan running… refresh page</span>" if STATE["scan_running"] else ""
|
||||
cls_status = f"<div class='bar'><i style='width:{int(100*STATE['classify_done']/max(1,STATE['classify_total']))}%'></i></div><span class='muted'>classifying {STATE['classify_done']}/{STATE['classify_total']}…</span>" if STATE["classify_running"] else ""
|
||||
body = f"""
|
||||
<h1 class='grad'>File Librarian</h1><p class='muted'>Scanner → Extractor → Nimble → Safety → You approve → Reversible move. Nothing moves without your click.</p>
|
||||
<div class='grid'>
|
||||
<div class='card'><div class='n'>{STATE.get('files',0)}</div><div class='l'>Files indexed</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('classified',0)}</div><div class='l'>Classified</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('dupes',0)}</div><div class='l'>In dup groups</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('pending',0)}</div><div class='l'>Approved (dry)</div></div>
|
||||
<div class='card'><div class='n'>{STATE.get('moved',0)}</div><div class='l'>Moved</div></div>
|
||||
</div>{err}
|
||||
<div class='panel'><h2>1 · Scan a location</h2>{scan_status}
|
||||
<form method='post' action='/scan'><div class='row'>
|
||||
<div><label>Root path</label><input type='text' name='root' value='~/Downloads'></div>
|
||||
<div><label>Max files</label><input type='number' name='max_files' value='2000'></div>
|
||||
<div><label>Min size (bytes)</label><input type='number' name='min_size' value='0'></div>
|
||||
<div><label>Max size (bytes, 0=no cap)</label><input type='number' name='max_size' value='0'></div>
|
||||
<div><button type='submit'>SCAN →</button></div>
|
||||
</div></form></div>
|
||||
<div class='panel'><h2>2 · Hash + Classify (Nimble on nightmare)</h2>{cls_status}
|
||||
<form method='post' action='/hash'><div class='row'>
|
||||
<div><label>Hash batch limit</label><input type='number' name='limit' value='500'></div>
|
||||
<div><label>Max MB to hash per file</label><input type='number' name='max_mb' value='64'></div>
|
||||
<div><button type='submit' class='ghost'>HASH DUPES →</button></div>
|
||||
</div></form>
|
||||
<form method='post' action='/classify' style='margin-top:10px'><div class='row'>
|
||||
<div><label>Classify batch size</label><input type='number' name='limit' value='100'></div>
|
||||
<div><button type='submit' class='teal'>CLASSIFY →</button></div>
|
||||
</div></form></div>
|
||||
<div class='panel'><h2>Pipeline</h2><p class='muted'>scan → hash (deterministic, sha256) → preview extract (pdf/zip/text/images) → Nimble structured JSON (think:false) → safety validate → <a href='/plan' style='color:#9d8fff'>review plan</a> → apply (dry-run default) → immutable op log → one-click undo.</p></div>
|
||||
"""
|
||||
return Response(shell(body, "home"), mimetype="text/html")
|
||||
|
||||
@app.route("/scan", methods=["POST"])
|
||||
def scan_post():
|
||||
root = request.form.get("root", "~/Downloads")
|
||||
if STATE["scan_running"]:
|
||||
return redirect("/")
|
||||
threading.Thread(target=scan_thread, args=(root, int(request.form.get("max_files", 2000)),
|
||||
int(request.form.get("min_size", 0)), int(request.form.get("max_size", 0))), daemon=True).start()
|
||||
return redirect("/")
|
||||
|
||||
@app.route("/hash", methods=["POST"])
|
||||
def hash_post():
|
||||
if not STATE["classify_running"]:
|
||||
threading.Thread(target=hash_thread, args=(int(request.form.get("limit", 500)),
|
||||
int(request.form.get("max_mb", 64))), daemon=True).start()
|
||||
return redirect("/")
|
||||
|
||||
@app.route("/classify", methods=["POST"])
|
||||
def classify_post():
|
||||
if not STATE["classify_running"]:
|
||||
threading.Thread(target=classify_thread, args=(int(request.form.get("limit", 100)),), daemon=True).start()
|
||||
return redirect("/")
|
||||
|
||||
@app.route("/files")
|
||||
def files_page():
|
||||
q = request.args.get("q", ""); cls = request.args.get("cls", "")
|
||||
c = db()
|
||||
sql = "SELECT * FROM files WHERE 1"; args = []
|
||||
if q: sql += " AND (path LIKE ? OR name LIKE ?)"; args += [f"%{q}%", f"%{q}%"]
|
||||
if cls: sql += " AND file_class=?"; args.append(cls)
|
||||
sql += " ORDER BY id DESC LIMIT 400"
|
||||
rows = c.execute(sql, args).fetchall()
|
||||
c.close()
|
||||
trs = ""
|
||||
for r in rows:
|
||||
cls_chip = f"<span class='chip c-{r['file_class']}'>{r['file_class'] or '—'}</span>" if r["file_class"] else "<span class='muted'>unclassified</span>"
|
||||
subj = r["subject"] or ""
|
||||
conf = f"{int((r['confidence'] or 0)*100)}%" if r["confidence"] is not None else ""
|
||||
dup = " ◆dup" if r["sha256"] and r["sha256"] not in ("TOOBIG","UNREADABLE") else ""
|
||||
pv = (r["preview"] or "")[:110]
|
||||
trs += (f"<tr><td>{cls_chip} {subj} {conf}{dup}</td>"
|
||||
f"<td><div style='word-break:break-all'>{r['path']}</div><div class='muted' style='font-size:11px'>{pv}</div></td>"
|
||||
f"<td class='muted'>{r['status']}</td></tr>")
|
||||
body = f"""
|
||||
<h1 class='grad'>Indexed Files</h1>
|
||||
<form method='get' action='/files'><div class='row'>
|
||||
<div><label>Search</label><input type='text' name='q' value='{q}'></div>
|
||||
<div><label>Class</label><input type='text' name='cls' value='{cls}'></div>
|
||||
<div><button type='submit' class='ghost'>FILTER</button></div>
|
||||
</div></form>
|
||||
<div class='panel'><table><tr><th>Class</th><th>Path + preview</th><th>Status</th></tr>{trs}</table></div>
|
||||
"""
|
||||
return Response(shell(body, "files"), mimetype="text/html")
|
||||
|
||||
@app.route("/plan")
|
||||
def plan_page():
|
||||
tx = load_taxonomy()
|
||||
min_conf = float(request.args.get("min_conf", 0.6))
|
||||
plans = build_plan(tx, min_conf)
|
||||
plans.sort(key=lambda p: (-p["conf"], p["verdict"] != "APPROVE"))
|
||||
cards = ""
|
||||
approved_default = 0
|
||||
for p in plans[:600]:
|
||||
checked = "checked" if p["verdict"] == "APPROVE" else ""
|
||||
if p["verdict"] == "APPROVE": approved_default += 1
|
||||
dupnote = "<span class='chip' style='background:rgba(255,180,84,.16);color:#ffcf8f'>DUPLICATE GROUP</span>" if p["dup"] else ""
|
||||
cards += (f"<div class='plan'>"
|
||||
f"<div class='src'>{p['path']}</div><div class='arrow'>↓</div><div class='dst'>{p['dst']}</div>"
|
||||
f"<div class='meta'><span class='chip c-{p['cls']}'>{p['cls']}</span><span class='muted'>{p['subj']} · {p['imp']}</span>"
|
||||
f"<span class='muted'>conf {int(p['conf']*100)}%</span>"
|
||||
f"<span class='v-{p['verdict']}' style='font-weight:700'>{p['verdict']}</span>"
|
||||
+ (f"<span class='muted'>({p['why']})</span>" if p["why"] else "") + dupnote +
|
||||
f"<span style='flex:1'></span>"
|
||||
f"<label style='display:inline;margin:0'><input type='checkbox' class='pick' data-id='{p['id']}' {checked} style='width:auto'> include</label></div></div>")
|
||||
body = f"""
|
||||
<h1 class='grad'>Organization Plan</h1>
|
||||
<form method='get' action='/plan'><div class='row'>
|
||||
<div><label>Min confidence (safety gate)</label><input type='number' step='0.05' name='min_conf' value='{min_conf}'></div>
|
||||
<div><button type='submit' class='ghost'>RE-GATE</button></div>
|
||||
</div></form>
|
||||
<p class='muted'>{len(plans)} proposals · {approved_default} auto-approved at this threshold. Uncheck anything you're unsure of — nothing moves without your click.</p>
|
||||
<div style='margin:12px 0'><button class='teal' onclick='applyMoves(true)'>DRY RUN (approve only)</button>
|
||||
<button onclick='applyMoves(false)'>APPLY CHECKED MOVES</button></div>
|
||||
<div id='plans'>{cards}</div>
|
||||
<div class='panel'><h2>Progress</h2><div class='bar'><i id='pbar' style='width:0%'></i></div><span class='muted' id='pmsg'></span></div>
|
||||
<script>
|
||||
function applyMoves(dry) {{
|
||||
var ids = [];
|
||||
document.querySelectorAll('.pick').forEach(function(cb) {{ if (cb.checked) ids.push(cb.dataset.id); }});
|
||||
if (!ids.length) {{ alert('nothing checked'); return; }}
|
||||
fetch('/api/apply', {{ method:'POST', headers:{{'Content-Type':'application/json'}},
|
||||
body: JSON.stringify({{ ids: ids, dry_run: dry }}) }})
|
||||
.then(function(r){{ return r.json(); }})
|
||||
.then(function(d) {{ document.getElementById('pmsg').textContent = 'done: ' + d.done + ' applied, ' + d.skipped + ' skipped, session ' + d.session; }});
|
||||
}}
|
||||
</script>
|
||||
"""
|
||||
return Response(shell(body, "plan"), mimetype="text/html")
|
||||
|
||||
@app.route("/api/apply", methods=["POST"])
|
||||
def api_apply():
|
||||
d = request.get_json(force=True)
|
||||
ids = [int(x) for x in d.get("ids", [])]
|
||||
done, skipped, session = apply_moves(ids, dry_run=bool(d.get("dry_run", True)))
|
||||
return {"done": done, "skipped": skipped, "session": session}
|
||||
|
||||
@app.route("/dupes")
|
||||
def dupes_page():
|
||||
c = db()
|
||||
groups = c.execute("""SELECT sha256, COUNT(*) n, SUM(size) b FROM files
|
||||
WHERE sha256 IS NOT NULL AND sha256 NOT IN ('TOOBIG','UNREADABLE')
|
||||
GROUP BY sha256 HAVING COUNT(*)>1 ORDER BY b DESC LIMIT 100""").fetchall()
|
||||
c.close()
|
||||
out = ""
|
||||
reclaim = 0
|
||||
for g in groups:
|
||||
c = db()
|
||||
members = c.execute("SELECT * FROM files WHERE sha256=?", (g["sha256"],)).fetchall()
|
||||
c.close()
|
||||
reclaim += g["b"] * (g["n"] - 1)
|
||||
rows = ""
|
||||
for m in members:
|
||||
rows += (f"<div style='display:flex;gap:10px;align-items:center;padding:4px 0'>"
|
||||
f"<span style='flex:1;font-family:ui-monospace,Menlo,monospace;font-size:12px;word-break:break-all'>{m['path']}</span>"
|
||||
f"<button class='ghost keep-btn' data-keep='{m['id']}' style='padding:4px 10px;font-size:11px'>KEEP THIS → hold others</button></div>")
|
||||
out += f"<div class='dupe'><b style='color:#ffcf8f'>{g['n']} copies · {g['b']/1e6:.1f} MB each</b>{rows}</div>"
|
||||
body = f"""
|
||||
<h1 class='grad'>Duplicates</h1>
|
||||
<p class='muted'>sha256-verified only — never guessed. Keepers stay; the rest go to <code>~/libra/dupe_hold/</code> (reversible). {reclaim/1e9:.2f} GB reclaimable.</p>
|
||||
<div>{out or "<p class='muted'>No duplicate groups yet — run Hash on the dashboard.</p>"}</div>
|
||||
<script>
|
||||
document.querySelectorAll('.keep-btn').forEach(function(b){{
|
||||
b.addEventListener('click', function() {{
|
||||
fetch('/api/keep', {{method:'POST',headers:{{'Content-Type':'application/json'}},body:JSON.stringify({{keep: parseInt(b.dataset.keep)}})}})
|
||||
.then(function(r){{return r.json()}}).then(function(d){{ b.textContent='held ' + d.held; b.disabled=true; }});
|
||||
}});
|
||||
}});
|
||||
</script>"""
|
||||
return Response(shell(body, "dupes"), mimetype="text/html")
|
||||
|
||||
@app.route("/api/keep", methods=["POST"])
|
||||
def api_keep():
|
||||
d = request.get_json(force=True)
|
||||
n = delete_duplicates(int(d["keep"]))
|
||||
return {"held": n}
|
||||
|
||||
@app.route("/sessions")
|
||||
def sessions_page():
|
||||
c = db()
|
||||
sess = c.execute("SELECT * FROM sessions ORDER BY ts DESC LIMIT 50").fetchall()
|
||||
ops = c.execute("SELECT * FROM operations ORDER BY ts DESC LIMIT 200").fetchall()
|
||||
c.close()
|
||||
sr = ""
|
||||
for s in sess:
|
||||
undo_btn = "" if s["undone"] or s["note"] == "duplicates to hold" else f"<button class='bad' onclick='undo(\"{s['id']}\")' style='padding:5px 12px;font-size:12px'>UNDO ALL ({s['n']})</button>"
|
||||
sr += (f"<div class='plan' style='display:flex;gap:14px;align-items:center'><b style='font-family:monospace'>{s['id']}</b>"
|
||||
f"<span class='muted'>{time.strftime('%b %d %H:%M', time.localtime(s['ts']))} · {s['n']} ops · {s['note']}"
|
||||
+ (" · <span style='color:#ff9c9c'>UNDONE</span>" if s["undone"] else "") + "</span>"
|
||||
f"<span style='flex:1'></span>{undo_btn}</div>")
|
||||
orows = ""
|
||||
for o in ops:
|
||||
orows += (f"<tr><td class='muted'>{time.strftime('%m-%d %H:%M', time.localtime(o['ts']))}</td><td>{o['op']}</td>"
|
||||
f"<td style='font-family:monospace;font-size:11px;word-break:break-all'>{o['src']}</td>"
|
||||
f"<td style='font-family:monospace;font-size:11px;word-break:break-all'>{o['dst']}</td>"
|
||||
f"<td class='muted'>{o['session']}" + (" ↩" if o["undone"] else "") + "</td></tr>")
|
||||
body = f"""
|
||||
<h1 class='grad'>Sessions · Operation Log</h1>
|
||||
<p class='muted'>Every move is logged with source, destination and sha256. Undo restores files exactly.</p>
|
||||
<div class='panel'><h2>Sessions</h2>{sr or '<span class=muted>no sessions yet</span>'}</div>
|
||||
<div class='panel'><h2>Operation log</h2><table><tr><th>When</th><th>Op</th><th>Source</th><th>Destination</th><th>Session</th></tr>{orows}</table></div>
|
||||
<script>
|
||||
function undo(sid) {{
|
||||
fetch('/api/undo', {{method:'POST',headers:{{'Content-Type':'application/json'}},body:JSON.stringify({{session: sid}})}})
|
||||
.then(function(r){{return r.json()}}).then(function(d){{ location.reload(); }});
|
||||
}}
|
||||
</script>"""
|
||||
return Response(shell(body, "sessions"), mimetype="text/html")
|
||||
|
||||
@app.route("/api/undo", methods=["POST"])
|
||||
def api_undo():
|
||||
d = request.get_json(force=True)
|
||||
n = undo_session(d["session"])
|
||||
return {"undone": n}
|
||||
|
||||
@app.route("/settings")
|
||||
def settings_page():
|
||||
tx = open(CFG_PATH).read()
|
||||
body = f"""
|
||||
<h1 class='grad'>Settings</h1>
|
||||
<div class='panel'><h2>Taxonomy (destination map)</h2>
|
||||
<form method='post' action='/settings/save'>
|
||||
<label>taxonomy.yaml — organized_root + class+subject → destination</label>
|
||||
<textarea name='tax'>{tx}</textarea>
|
||||
<div style='margin-top:10px'><button type='submit' class='teal'>SAVE TAXONOMY</button></div>
|
||||
</form></div>
|
||||
<div class='panel'><h2>Engine</h2>
|
||||
<p class='muted'>Model: <code>{MODEL}</code> @ <code>{OLLAMA}</code> (Nimble decision lane, think:false) · DB: <code>{DB_PATH}</code></p>
|
||||
<p class='muted'>Safety gates: no system paths · no symlinks · no overwrite (auto-suffix) · confidence gate on plan · every move logged + reversible · dupes go to hold, never deleted.</p></div>
|
||||
"""
|
||||
return Response(shell(body, "settings"), mimetype="text/html")
|
||||
|
||||
@app.route("/settings/save", methods=["POST"])
|
||||
def settings_save():
|
||||
open(CFG_PATH, "w").write(request.form.get("tax", ""))
|
||||
return redirect("/settings")
|
||||
|
||||
if __name__ == "__main__":
|
||||
init_db()
|
||||
refresh_counts()
|
||||
app.run(host="127.0.0.1", port=5731, debug=False)
|
||||
Reference in New Issue
Block a user