seromy's picture
Upload batch.py with huggingface_hub
a7cdf62 verified
Raw History Blame Contribute Delete
5.88 kB
"""Resumable local folder classification. Originals are never moved or modified."""
import argparse
import csv
import hashlib
import json
import os
from pathlib import Path
import time
from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent
from PIL import Image
FIELDS = ["filename", "label", "document_probability", "selected_probability", "needs_review", "elapsed_seconds", "error"]
def export_csv(rows, destination):
"""Atomically publish an Excel-friendly CSV snapshot."""
destination = Path(destination)
temporary = destination.with_suffix(".tmp")
with temporary.open("w", encoding="utf-8-sig", newline="") as f:
writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore")
writer.writeheader()
for row in rows:
safe = dict(row)
for field in ("filename", "error"):
value = str(safe.get(field, ""))
safe[field] = "'" + value if value.startswith(("=", "+", "-", "@")) else value
writer.writerow(safe)
temporary.replace(destination)
def run(folder, output, threshold=0.5, review_confidence=0.8, limit=None, device=None):
folder, output = Path(folder).resolve(), Path(output).resolve()
output.mkdir(parents=True, exist_ok=True)
source = Path(__file__).resolve().parent
skip_source = not folder.is_relative_to(source)
files = sorted(p for p in folder.rglob("*") if p.is_file() and p.suffix.lower() in EXTENSIONS
and output not in p.parents and (not skip_source or source not in p.parents)
and not any(x.startswith('.') for x in p.relative_to(folder).parts))
if limit:
files = files[:limit]
settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION,
"threshold": threshold, "review_confidence": review_confidence, "folder": str(folder)}
config_path = output / "settings.json"
if config_path.exists() and json.loads(config_path.read_text()) != settings:
raise ValueError("Output folder belongs to different settings. Choose a new --output folder.")
config_path.write_text(json.dumps(settings, indent=2))
journal = output / "predictions.jsonl"
previous = {}
if journal.exists():
for line in journal.read_text().splitlines():
try:
row = json.loads(line)
previous[row["filename"]] = row
except (json.JSONDecodeError, KeyError):
continue # A power loss can leave the last journal line incomplete.
model = None
started, processed = time.monotonic(), 0
rows = []
# Repair a partial final line before appending any new durable records.
if journal.exists() and journal.stat().st_size:
with journal.open("rb+") as f:
data = f.read()
if not data.endswith(b"\n"):
f.truncate(data.rfind(b"\n") + 1)
with journal.open("a", encoding="utf-8") as log:
for index, path in enumerate(files, 1):
name = path.relative_to(folder).as_posix()
fingerprint = hashlib.sha256(path.read_bytes()).hexdigest()
old = previous.get(name)
if old and old.get("sha256") == fingerprint and not old.get("error"):
rows.append(old)
continue
if model is None:
model = load_agent(device)
try:
with Image.open(path) as image:
result = classify(image, threshold, review_confidence, model)
row = {"filename": name, "sha256": fingerprint, **result, "error": ""}
except Exception as exc:
row = {"filename": name, "sha256": fingerprint, "label": "error", "needs_review": True,
"error": f"{type(exc).__name__}: {exc}"}
log.write(json.dumps(row, ensure_ascii=False) + "\n")
log.flush()
os.fsync(log.fileno())
rows.append(row)
processed += 1
if processed % 25 == 0 or index == len(files) or row.get("error"):
export_csv(rows, output / "results.csv")
speed = (time.monotonic() - started) / processed
remaining = len(files) - index
print(f"{index}/{len(files)} | {speed:.2f}s/image | ETA {remaining*speed/60:.1f}min | {name}: {row['label']}", flush=True)
export_csv(rows, output / "results.csv")
summary = {"total": len(rows), "documents": sum(r["label"] == "document" for r in rows),
"non_documents": sum(r["label"] == "non_document" for r in rows),
"errors": sum(r["label"] == "error" for r in rows),
"needs_review": sum(bool(r.get("needs_review")) for r in rows),
"newly_processed": processed, "wall_seconds": round(time.monotonic()-started, 2)}
(output / "summary.json").write_text(json.dumps(summary, indent=2))
print(json.dumps(summary), flush=True)
return summary
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("folder", type=Path)
parser.add_argument("--output", type=Path, default=Path("document-results"))
parser.add_argument("--threshold", type=float, default=0.5)
parser.add_argument("--review-confidence", type=float, default=0.8)
parser.add_argument("--limit", type=int)
parser.add_argument("--device", choices=["cpu", "mps", "cuda"])
args = parser.parse_args()
if not 0 < args.threshold < 1 or not 0.5 <= args.review_confidence <= 1:
parser.error("threshold must be between 0 and 1; review confidence between 0.5 and 1")
if args.limit is not None and args.limit < 1:
parser.error("limit must be positive")
run(args.folder, args.output, args.threshold, args.review_confidence, args.limit, args.device)