Spaces:
Running on Zero
Running on Zero
File size: 5,880 Bytes
cff910f a7cdf62 cff910f a7cdf62 cff910f a7cdf62 cff910f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """Resumable local folder classification. Originals are never moved or modified."""
import argparse
import csv
import hashlib
import json
import os
from pathlib import Path
import time
from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent
from PIL import Image
FIELDS = ["filename", "label", "document_probability", "selected_probability", "needs_review", "elapsed_seconds", "error"]
def export_csv(rows, destination):
"""Atomically publish an Excel-friendly CSV snapshot."""
destination = Path(destination)
temporary = destination.with_suffix(".tmp")
with temporary.open("w", encoding="utf-8-sig", newline="") as f:
writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore")
writer.writeheader()
for row in rows:
safe = dict(row)
for field in ("filename", "error"):
value = str(safe.get(field, ""))
safe[field] = "'" + value if value.startswith(("=", "+", "-", "@")) else value
writer.writerow(safe)
temporary.replace(destination)
def run(folder, output, threshold=0.5, review_confidence=0.8, limit=None, device=None):
folder, output = Path(folder).resolve(), Path(output).resolve()
output.mkdir(parents=True, exist_ok=True)
source = Path(__file__).resolve().parent
skip_source = not folder.is_relative_to(source)
files = sorted(p for p in folder.rglob("*") if p.is_file() and p.suffix.lower() in EXTENSIONS
and output not in p.parents and (not skip_source or source not in p.parents)
and not any(x.startswith('.') for x in p.relative_to(folder).parts))
if limit:
files = files[:limit]
settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION,
"threshold": threshold, "review_confidence": review_confidence, "folder": str(folder)}
config_path = output / "settings.json"
if config_path.exists() and json.loads(config_path.read_text()) != settings:
raise ValueError("Output folder belongs to different settings. Choose a new --output folder.")
config_path.write_text(json.dumps(settings, indent=2))
journal = output / "predictions.jsonl"
previous = {}
if journal.exists():
for line in journal.read_text().splitlines():
try:
row = json.loads(line)
previous[row["filename"]] = row
except (json.JSONDecodeError, KeyError):
continue # A power loss can leave the last journal line incomplete.
model = None
started, processed = time.monotonic(), 0
rows = []
# Repair a partial final line before appending any new durable records.
if journal.exists() and journal.stat().st_size:
with journal.open("rb+") as f:
data = f.read()
if not data.endswith(b"\n"):
f.truncate(data.rfind(b"\n") + 1)
with journal.open("a", encoding="utf-8") as log:
for index, path in enumerate(files, 1):
name = path.relative_to(folder).as_posix()
fingerprint = hashlib.sha256(path.read_bytes()).hexdigest()
old = previous.get(name)
if old and old.get("sha256") == fingerprint and not old.get("error"):
rows.append(old)
continue
if model is None:
model = load_agent(device)
try:
with Image.open(path) as image:
result = classify(image, threshold, review_confidence, model)
row = {"filename": name, "sha256": fingerprint, **result, "error": ""}
except Exception as exc:
row = {"filename": name, "sha256": fingerprint, "label": "error", "needs_review": True,
"error": f"{type(exc).__name__}: {exc}"}
log.write(json.dumps(row, ensure_ascii=False) + "\n")
log.flush()
os.fsync(log.fileno())
rows.append(row)
processed += 1
if processed % 25 == 0 or index == len(files) or row.get("error"):
export_csv(rows, output / "results.csv")
speed = (time.monotonic() - started) / processed
remaining = len(files) - index
print(f"{index}/{len(files)} | {speed:.2f}s/image | ETA {remaining*speed/60:.1f}min | {name}: {row['label']}", flush=True)
export_csv(rows, output / "results.csv")
summary = {"total": len(rows), "documents": sum(r["label"] == "document" for r in rows),
"non_documents": sum(r["label"] == "non_document" for r in rows),
"errors": sum(r["label"] == "error" for r in rows),
"needs_review": sum(bool(r.get("needs_review")) for r in rows),
"newly_processed": processed, "wall_seconds": round(time.monotonic()-started, 2)}
(output / "summary.json").write_text(json.dumps(summary, indent=2))
print(json.dumps(summary), flush=True)
return summary
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("folder", type=Path)
parser.add_argument("--output", type=Path, default=Path("document-results"))
parser.add_argument("--threshold", type=float, default=0.5)
parser.add_argument("--review-confidence", type=float, default=0.8)
parser.add_argument("--limit", type=int)
parser.add_argument("--device", choices=["cpu", "mps", "cuda"])
args = parser.parse_args()
if not 0 < args.threshold < 1 or not 0.5 <= args.review_confidence <= 1:
parser.error("threshold must be between 0 and 1; review confidence between 0.5 and 1")
if args.limit is not None and args.limit < 1:
parser.error("limit must be positive")
run(args.folder, args.output, args.threshold, args.review_confidence, args.limit, args.device)
|