"""Resumable local folder classification. Originals are never moved or modified.""" import argparse import csv import hashlib import json import os from pathlib import Path import time from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent from PIL import Image FIELDS = ["filename", "label", "document_probability", "selected_probability", "needs_review", "elapsed_seconds", "error"] def export_csv(rows, destination): """Atomically publish an Excel-friendly CSV snapshot.""" destination = Path(destination) temporary = destination.with_suffix(".tmp") with temporary.open("w", encoding="utf-8-sig", newline="") as f: writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore") writer.writeheader() for row in rows: safe = dict(row) for field in ("filename", "error"): value = str(safe.get(field, "")) safe[field] = "'" + value if value.startswith(("=", "+", "-", "@")) else value writer.writerow(safe) temporary.replace(destination) def run(folder, output, threshold=0.5, review_confidence=0.8, limit=None, device=None): folder, output = Path(folder).resolve(), Path(output).resolve() output.mkdir(parents=True, exist_ok=True) source = Path(__file__).resolve().parent skip_source = not folder.is_relative_to(source) files = sorted(p for p in folder.rglob("*") if p.is_file() and p.suffix.lower() in EXTENSIONS and output not in p.parents and (not skip_source or source not in p.parents) and not any(x.startswith('.') for x in p.relative_to(folder).parts)) if limit: files = files[:limit] settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION, "threshold": threshold, "review_confidence": review_confidence, "folder": str(folder)} config_path = output / "settings.json" if config_path.exists() and json.loads(config_path.read_text()) != settings: raise ValueError("Output folder belongs to different settings. Choose a new --output folder.") config_path.write_text(json.dumps(settings, indent=2)) journal = output / "predictions.jsonl" previous = {} if journal.exists(): for line in journal.read_text().splitlines(): try: row = json.loads(line) previous[row["filename"]] = row except (json.JSONDecodeError, KeyError): continue # A power loss can leave the last journal line incomplete. model = None started, processed = time.monotonic(), 0 rows = [] # Repair a partial final line before appending any new durable records. if journal.exists() and journal.stat().st_size: with journal.open("rb+") as f: data = f.read() if not data.endswith(b"\n"): f.truncate(data.rfind(b"\n") + 1) with journal.open("a", encoding="utf-8") as log: for index, path in enumerate(files, 1): name = path.relative_to(folder).as_posix() fingerprint = hashlib.sha256(path.read_bytes()).hexdigest() old = previous.get(name) if old and old.get("sha256") == fingerprint and not old.get("error"): rows.append(old) continue if model is None: model = load_agent(device) try: with Image.open(path) as image: result = classify(image, threshold, review_confidence, model) row = {"filename": name, "sha256": fingerprint, **result, "error": ""} except Exception as exc: row = {"filename": name, "sha256": fingerprint, "label": "error", "needs_review": True, "error": f"{type(exc).__name__}: {exc}"} log.write(json.dumps(row, ensure_ascii=False) + "\n") log.flush() os.fsync(log.fileno()) rows.append(row) processed += 1 if processed % 25 == 0 or index == len(files) or row.get("error"): export_csv(rows, output / "results.csv") speed = (time.monotonic() - started) / processed remaining = len(files) - index print(f"{index}/{len(files)} | {speed:.2f}s/image | ETA {remaining*speed/60:.1f}min | {name}: {row['label']}", flush=True) export_csv(rows, output / "results.csv") summary = {"total": len(rows), "documents": sum(r["label"] == "document" for r in rows), "non_documents": sum(r["label"] == "non_document" for r in rows), "errors": sum(r["label"] == "error" for r in rows), "needs_review": sum(bool(r.get("needs_review")) for r in rows), "newly_processed": processed, "wall_seconds": round(time.monotonic()-started, 2)} (output / "summary.json").write_text(json.dumps(summary, indent=2)) print(json.dumps(summary), flush=True) return summary if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("folder", type=Path) parser.add_argument("--output", type=Path, default=Path("document-results")) parser.add_argument("--threshold", type=float, default=0.5) parser.add_argument("--review-confidence", type=float, default=0.8) parser.add_argument("--limit", type=int) parser.add_argument("--device", choices=["cpu", "mps", "cuda"]) args = parser.parse_args() if not 0 < args.threshold < 1 or not 0.5 <= args.review_confidence <= 1: parser.error("threshold must be between 0 and 1; review confidence between 0.5 and 1") if args.limit is not None and args.limit < 1: parser.error("limit must be positive") run(args.folder, args.output, args.threshold, args.review_confidence, args.limit, args.device)