[NOTICKET] feat(knowledge_extraction): consume the parsing half's ParsedDocument
Browse filesWires the adapter to the contract Sofhia landed in src/knowledge_parsing, so the
seam is exercised with the real types on both sides rather than a hand-written
dict.
- accepts a ParsedDocument instance directly, not just a dict
- page_idx / page_idxs / page_idx_end are now the primary field names; the older
draft and prototype names still load so existing fixtures keep working
- page numbers pass through 0-BASED with no conversion anywhere. Converting is
the review UI's job, done once at display time — an off-by-one here would be
invisible until an expert opened the wrong page
- parser_backend is folded into parser_version, because the same MinerU build
emits different text from `pipeline` and `vlm`; a shift in extraction output
has to stay attributable to one or the other
New local seam tests import src.knowledge_parsing.contracts directly, so a field
rename on either side fails there instead of silently degrading a corpus run.
They pin the three properties the seam exists to guarantee: 0-based pages
survive, parser identity survives, and the literal section wording survives —
"Physical of Availability (PA)" from the heading AND "Physical Availability (PA)"
from the intro both reach the extraction half, which is what lets the
discrepancy be put to the expert.
Sofhia's call to keep the heading OUT of Chunk.text is confirmed correct from
this side: the extraction half already composes heading + text for the span
haystack and already treats a heading naming the term as a mention at offset 0.
Putting it in the text would have made it a real mention and inflated
mention_count — the exact number compared against the frozen baseline.
Verification: ruff clean; import main OK; full suite 494 passed, 7 skipped
(488 + 6 new seam tests).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
|
@@ -26,9 +26,17 @@ from typing import Any
|
|
| 26 |
|
| 27 |
from .models import Chunk, ParsedDoc
|
| 28 |
|
| 29 |
-
#
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
|
| 34 |
def chunk_from_dict(raw: dict[str, Any], doc_id: str, ordinal: int = 0) -> Chunk:
|
|
@@ -42,7 +50,9 @@ def chunk_from_dict(raw: dict[str, Any], doc_id: str, ordinal: int = 0) -> Chunk
|
|
| 42 |
page_start = _first(raw, _PAGE_KEYS)
|
| 43 |
if page_start is None:
|
| 44 |
page_start = min(pages) if pages else 0
|
| 45 |
-
page_end =
|
|
|
|
|
|
|
| 46 |
|
| 47 |
return Chunk(
|
| 48 |
chunk_id=raw.get("chunk_id") or f"{doc_id}#{ordinal:04d}",
|
|
@@ -73,12 +83,21 @@ def parsed_doc_from_artifact(
|
|
| 73 |
envelope lands, its `content_hash`/`n_pages`/`version` are preferred over
|
| 74 |
the values derived here.
|
| 75 |
"""
|
|
|
|
|
|
|
|
|
|
| 76 |
if isinstance(artifact, dict):
|
| 77 |
items = artifact.get("chunks") or []
|
| 78 |
doc_id = doc_id or artifact.get("doc_id")
|
| 79 |
source_ref = source_ref or artifact.get("source_path") or artifact.get("source_ref") or ""
|
| 80 |
parser_name = artifact.get("parser_name") or parser_name
|
| 81 |
parser_version = artifact.get("parser_version") or parser_version
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
declared_hash = artifact.get("content_hash")
|
| 83 |
declared_pages = artifact.get("n_pages")
|
| 84 |
else:
|
|
|
|
| 26 |
|
| 27 |
from .models import Chunk, ParsedDoc
|
| 28 |
|
| 29 |
+
# Field names accepted for the same concept. `page_idx`/`page_idxs` are the
|
| 30 |
+
# parsing half's contract; the rest are earlier drafts and the prototype's shape,
|
| 31 |
+
# kept so old fixtures still load.
|
| 32 |
+
#
|
| 33 |
+
# Page numbers are 0-BASED throughout, exactly as the parser reports them. No
|
| 34 |
+
# conversion happens anywhere in this pipeline: converting to 1-based is the
|
| 35 |
+
# review UI's job, done once at display time. An off-by-one here would be
|
| 36 |
+
# invisible until an expert opened the wrong page.
|
| 37 |
+
_PAGE_KEYS = ("page_idx", "page_start", "page")
|
| 38 |
+
_PAGES_KEYS = ("page_idxs", "pages", "page_list")
|
| 39 |
+
_PAGE_END_KEYS = ("page_idx_end", "page_end")
|
| 40 |
|
| 41 |
|
| 42 |
def chunk_from_dict(raw: dict[str, Any], doc_id: str, ordinal: int = 0) -> Chunk:
|
|
|
|
| 50 |
page_start = _first(raw, _PAGE_KEYS)
|
| 51 |
if page_start is None:
|
| 52 |
page_start = min(pages) if pages else 0
|
| 53 |
+
page_end = _first(raw, _PAGE_END_KEYS)
|
| 54 |
+
if page_end is None:
|
| 55 |
+
page_end = max(pages) if pages else page_start
|
| 56 |
|
| 57 |
return Chunk(
|
| 58 |
chunk_id=raw.get("chunk_id") or f"{doc_id}#{ordinal:04d}",
|
|
|
|
| 83 |
envelope lands, its `content_hash`/`n_pages`/`version` are preferred over
|
| 84 |
the values derived here.
|
| 85 |
"""
|
| 86 |
+
if hasattr(artifact, "model_dump"): # a ParsedDocument from the parsing half
|
| 87 |
+
artifact = artifact.model_dump(mode="json")
|
| 88 |
+
|
| 89 |
if isinstance(artifact, dict):
|
| 90 |
items = artifact.get("chunks") or []
|
| 91 |
doc_id = doc_id or artifact.get("doc_id")
|
| 92 |
source_ref = source_ref or artifact.get("source_path") or artifact.get("source_ref") or ""
|
| 93 |
parser_name = artifact.get("parser_name") or parser_name
|
| 94 |
parser_version = artifact.get("parser_version") or parser_version
|
| 95 |
+
# The backend matters as much as the version: the same MinerU build can
|
| 96 |
+
# emit different text from `pipeline` and `vlm`, so a shift in extraction
|
| 97 |
+
# output has to be attributable to one or the other.
|
| 98 |
+
backend = artifact.get("parser_backend")
|
| 99 |
+
if backend:
|
| 100 |
+
parser_version = f"{parser_version}/{backend}" if parser_version else backend
|
| 101 |
declared_hash = artifact.get("content_hash")
|
| 102 |
declared_pages = artifact.get("n_pages")
|
| 103 |
else:
|