File size: 5,344 Bytes
5eb40dc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""The confidence-reporting bundle.

Chosen policy: when the agent is not fully confident it never blocks and never
fails -- it emits every plausible artifact plus a manifest saying what it was
unsure about.  A visitor who does not stay to answer a question still leaves
with usable output.
"""

from __future__ import annotations

import json
import logging
import os
import zipfile
from pathlib import Path

from ..state import Document
from . import writers

log = logging.getLogger("docagent.export.bundle")

__all__ = ["build_manifest", "write_manifest", "build_bundle", "LOW_CONFIDENCE"]

#: Pages below this are called out in the manifest and the chat reply.
LOW_CONFIDENCE = 0.70


def build_manifest(doc: Document) -> dict:
    """Per-page provenance and an explicit list of what to double-check."""
    low = [p for p in doc.pages if 0.0 < p.kind_confidence < LOW_CONFIDENCE]
    unread = [p for p in doc.pages if not p.text().strip()]

    concerns: list[str] = []
    for p in low:
        concerns.append(
            "page %d read as %s with confidence %.2f -- verify before relying on it"
            % (p.page_no, p.kind.value, p.kind_confidence)
        )
    for p in unread:
        concerns.append("page %d produced no text" % p.page_no)
    concerns.extend(doc.warnings)

    return {
        "sources": doc.sources,
        "instruction": doc.instruction,
        "page_count": doc.n_pages,
        "overall_type": doc.dominant_kind().value,
        "min_page_confidence": round(doc.min_confidence(), 4),
        "tables_found": len(doc.all_tables()),
        "fields_found": len(doc.all_fields()),
        "needs_review": bool(concerns),
        "concerns": concerns,
        "pages": [
            {
                "page": p.page_no,
                "source": p.source,
                "type": p.kind.value,
                "confidence": round(p.kind_confidence, 4),
                "backend": p.backend or "none",
                "has_text_layer": p.has_text_layer,
                "chars": len(p.text()),
                "tables": len(p.tables),
                "fields": len(p.fields),
                "warnings": p.warnings,
            }
            for p in doc.pages
        ],
        "artifacts": [
            {"file": os.path.basename(a.path), "kind": a.kind, "label": a.label,
             "detail": a.detail}
            for a in doc.artifacts
        ],
        "trace": doc.trace,
    }


def write_manifest(doc: Document, directory: str | os.PathLike,
                   name: str = "manifest.json") -> str:
    path = Path(directory)
    path.mkdir(parents=True, exist_ok=True)
    target = path / name
    target.write_text(
        json.dumps(build_manifest(doc), indent=2, ensure_ascii=False), encoding="utf-8"
    )
    return str(target)


def build_bundle(
    doc: Document,
    directory: str | os.PathLike,
    name: str = "output.zip",
    *,
    include_searchable_pdf: bool = True,
) -> str:
    """Write every artifact that makes sense for this document, then zip it.

    Writers are attempted independently: a failure in one format is recorded as
    a warning and the rest of the bundle is still produced.
    """
    directory = Path(directory)
    directory.mkdir(parents=True, exist_ok=True)
    existing = {os.path.abspath(a.path) for a in doc.artifacts}

    def attempt(kind: str, label: str, fn, detail: str = "") -> None:
        try:
            produced = fn()
        except Exception as exc:
            doc.warnings.append("could not write %s: %s" % (kind, exc))
            log.warning("writer %s failed: %s", kind, exc)
            return
        for p in produced if isinstance(produced, list) else [produced]:
            if os.path.abspath(p) not in existing:
                doc.add_artifact(p, kind, label, detail)
                existing.add(os.path.abspath(p))

    attempt("markdown", "Full text (Markdown)", lambda: writers.write_markdown(doc, directory))
    attempt("json", "Structured JSON", lambda: writers.write_json(doc, directory))

    if doc.all_tables():
        attempt("xlsx", "Tables (Excel)", lambda: writers.write_xlsx(doc, directory),
                "%d table(s)" % len(doc.all_tables()))
        attempt("csv", "Tables (CSV)", lambda: writers.write_csv(doc, directory))

    if doc.all_fields():
        attempt(
            "json", "Key fields",
            lambda: writers.write_fields_json(doc, directory),
            "%d field(s)" % len(doc.all_fields()),
        )

    attempt("docx", "Word document", lambda: writers.write_docx(doc, directory))
    attempt("pdf", "Summary report", lambda: writers.write_pdf_report(doc, directory))
    if include_searchable_pdf and any(p.image is not None for p in doc.pages):
        attempt("pdf", "Searchable PDF",
                lambda: writers.write_searchable_pdf(doc, directory))

    manifest = write_manifest(doc, directory)

    zip_path = directory / name
    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
        for artifact in doc.artifacts:
            if os.path.exists(artifact.path):
                zf.write(artifact.path, os.path.basename(artifact.path))
        zf.write(manifest, os.path.basename(manifest))

    doc.add_artifact(str(zip_path), "zip", "Everything (ZIP)",
                     "%d file(s)" % (len(doc.artifacts) + 1))
    return str(zip_path)