File size: 5,880 Bytes
cff910f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a7cdf62
 
cff910f
a7cdf62
 
cff910f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a7cdf62
cff910f
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
"""Resumable local folder classification. Originals are never moved or modified."""
import argparse
import csv
import hashlib
import json
import os
from pathlib import Path
import time
from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent
from PIL import Image

FIELDS = ["filename", "label", "document_probability", "selected_probability", "needs_review", "elapsed_seconds", "error"]


def export_csv(rows, destination):
    """Atomically publish an Excel-friendly CSV snapshot."""
    destination = Path(destination)
    temporary = destination.with_suffix(".tmp")
    with temporary.open("w", encoding="utf-8-sig", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore")
        writer.writeheader()
        for row in rows:
            safe = dict(row)
            for field in ("filename", "error"):
                value = str(safe.get(field, ""))
                safe[field] = "'" + value if value.startswith(("=", "+", "-", "@")) else value
            writer.writerow(safe)
    temporary.replace(destination)


def run(folder, output, threshold=0.5, review_confidence=0.8, limit=None, device=None):
    folder, output = Path(folder).resolve(), Path(output).resolve()
    output.mkdir(parents=True, exist_ok=True)
    source = Path(__file__).resolve().parent
    skip_source = not folder.is_relative_to(source)
    files = sorted(p for p in folder.rglob("*") if p.is_file() and p.suffix.lower() in EXTENSIONS
                   and output not in p.parents and (not skip_source or source not in p.parents)
                   and not any(x.startswith('.') for x in p.relative_to(folder).parts))
    if limit:
        files = files[:limit]
    settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION,
                "threshold": threshold, "review_confidence": review_confidence, "folder": str(folder)}
    config_path = output / "settings.json"
    if config_path.exists() and json.loads(config_path.read_text()) != settings:
        raise ValueError("Output folder belongs to different settings. Choose a new --output folder.")
    config_path.write_text(json.dumps(settings, indent=2))
    journal = output / "predictions.jsonl"
    previous = {}
    if journal.exists():
        for line in journal.read_text().splitlines():
            try:
                row = json.loads(line)
                previous[row["filename"]] = row
            except (json.JSONDecodeError, KeyError):
                continue  # A power loss can leave the last journal line incomplete.
    model = None
    started, processed = time.monotonic(), 0
    rows = []
    # Repair a partial final line before appending any new durable records.
    if journal.exists() and journal.stat().st_size:
        with journal.open("rb+") as f:
            data = f.read()
            if not data.endswith(b"\n"):
                f.truncate(data.rfind(b"\n") + 1)
    with journal.open("a", encoding="utf-8") as log:
        for index, path in enumerate(files, 1):
            name = path.relative_to(folder).as_posix()
            fingerprint = hashlib.sha256(path.read_bytes()).hexdigest()
            old = previous.get(name)
            if old and old.get("sha256") == fingerprint and not old.get("error"):
                rows.append(old)
                continue
            if model is None:
                model = load_agent(device)
            try:
                with Image.open(path) as image:
                    result = classify(image, threshold, review_confidence, model)
                row = {"filename": name, "sha256": fingerprint, **result, "error": ""}
            except Exception as exc:
                row = {"filename": name, "sha256": fingerprint, "label": "error", "needs_review": True,
                       "error": f"{type(exc).__name__}: {exc}"}
            log.write(json.dumps(row, ensure_ascii=False) + "\n")
            log.flush()
            os.fsync(log.fileno())
            rows.append(row)
            processed += 1
            if processed % 25 == 0 or index == len(files) or row.get("error"):
                export_csv(rows, output / "results.csv")
                speed = (time.monotonic() - started) / processed
                remaining = len(files) - index
                print(f"{index}/{len(files)} | {speed:.2f}s/image | ETA {remaining*speed/60:.1f}min | {name}: {row['label']}", flush=True)
    export_csv(rows, output / "results.csv")
    summary = {"total": len(rows), "documents": sum(r["label"] == "document" for r in rows),
               "non_documents": sum(r["label"] == "non_document" for r in rows),
               "errors": sum(r["label"] == "error" for r in rows),
               "needs_review": sum(bool(r.get("needs_review")) for r in rows),
               "newly_processed": processed, "wall_seconds": round(time.monotonic()-started, 2)}
    (output / "summary.json").write_text(json.dumps(summary, indent=2))
    print(json.dumps(summary), flush=True)
    return summary


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("folder", type=Path)
    parser.add_argument("--output", type=Path, default=Path("document-results"))
    parser.add_argument("--threshold", type=float, default=0.5)
    parser.add_argument("--review-confidence", type=float, default=0.8)
    parser.add_argument("--limit", type=int)
    parser.add_argument("--device", choices=["cpu", "mps", "cuda"])
    args = parser.parse_args()
    if not 0 < args.threshold < 1 or not 0.5 <= args.review_confidence <= 1:
        parser.error("threshold must be between 0 and 1; review confidence between 0.5 and 1")
    if args.limit is not None and args.limit < 1:
        parser.error("limit must be positive")
    run(args.folder, args.output, args.threshold, args.review_confidence, args.limit, args.device)