Claude commited on
Commit
b37aafd
·
unverified ·
1 Parent(s): 9bb793c

feat(sprint3-session-a): générateur ALTO v4 depuis PageMaster

Browse files

- export/alto.py : generate_alto(PageMaster) → XML ALTO v4 valide
- Source canonique : PageMaster exclusivement (R02)
- bbox [x,y,w,h] → HPOS/VPOS/WIDTH/HEIGHT (correspondance directe, R03)
- text_block / margin / rubric → TextBlock
- miniature / decorated_initial → Illustration (avec TYPE=)
- other → ComposedBlock
- OCR par région (ocr.blocks avec region_id) ou fallback diplomatic_text
dans le premier TextBlock
- OCRProcessing avec model_id, prompt_version, datetime si processing présent
- Région invalide → ValueError explicite (jamais ALTO partiel silencieux)
- write_alto() pour écriture dans data/corpora/{slug}/pages/{folio}/alto.xml
- test_export_alto.py : 46 tests (XML valide, bbox exactes, mapping types,
OCR par région, scénarios réalistes Beatus HR/BR et Grandes Chroniques)

Total : 208 tests passent, 3 skippés (intégration réseau).

https://claude.ai/code/session_018woyEHc8HG2th7V4ewJ4Kg

backend/app/services/export/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Services d'export documentaire — ALTO, METS, Manifest IIIF (Sprint 3).
3
+ """
4
+ from app.services.export.alto import generate_alto, write_alto
5
+
6
+ __all__ = [
7
+ "generate_alto",
8
+ "write_alto",
9
+ ]
backend/app/services/export/alto.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Générateur ALTO v4 depuis un PageMaster validé (R02).
3
+
4
+ Source canonique : PageMaster uniquement — jamais la réponse brute gemini_raw.json.
5
+ bbox [x, y, width, height] → HPOS / VPOS / WIDTH / HEIGHT (correspondance directe).
6
+
7
+ Mapping RegionType → élément ALTO :
8
+ text_block / margin / rubric → TextBlock
9
+ miniature / decorated_initial → Illustration
10
+ other → ComposedBlock
11
+ """
12
+ # 1. stdlib
13
+ import logging
14
+ from pathlib import Path
15
+
16
+ # 2. third-party
17
+ from lxml import etree
18
+ from pydantic import ValidationError
19
+
20
+ # 3. local
21
+ from app.schemas.page_master import OCRResult, PageMaster, Region, RegionType
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # ── Namespaces ALTO v4 ──────────────────────────────────────────────────────
26
+ _ALTO_NS = "http://www.loc.gov/standards/alto/ns-v4#"
27
+ _XSI_NS = "http://www.w3.org/2001/XMLSchema-instance"
28
+ _SCHEMA_LOC = (
29
+ "http://www.loc.gov/standards/alto/ns-v4# "
30
+ "https://www.loc.gov/standards/alto/v4/alto-4-2.xsd"
31
+ )
32
+
33
+ # ── Classification des types de régions ─────────────────────────────────────
34
+ _TEXT_REGION_TYPES = {RegionType.TEXT_BLOCK, RegionType.MARGIN, RegionType.RUBRIC}
35
+ _ILLUSTRATION_TYPES = {RegionType.MINIATURE, RegionType.DECORATED_INITIAL}
36
+ _COMPOSED_TYPES = {RegionType.OTHER}
37
+
38
+
39
+ def _a(tag: str) -> str:
40
+ """Retourne le tag qualifié dans le namespace ALTO."""
41
+ return f"{{{_ALTO_NS}}}{tag}"
42
+
43
+
44
+ def _bbox_attrib(region: Region) -> dict[str, str]:
45
+ """Retourne les attributs ALTO bbox depuis une Region (R03 → direct mapping)."""
46
+ x, y, w, h = region.bbox
47
+ return {"HPOS": str(x), "VPOS": str(y), "WIDTH": str(w), "HEIGHT": str(h)}
48
+
49
+
50
+ def _build_text_block(
51
+ parent: etree._Element,
52
+ region: Region,
53
+ ocr_block: dict | None,
54
+ fallback_text: str,
55
+ language: str,
56
+ confidence: float,
57
+ ) -> None:
58
+ """Ajoute un élément TextBlock au parent.
59
+
60
+ Si un bloc OCR est disponible pour cette région ou si fallback_text est fourni,
61
+ ajoute un TextLine / String enfant. Sinon, le TextBlock reste vide (valide ALTO).
62
+ """
63
+ attrib = {"ID": region.id, "LANG": language, **_bbox_attrib(region)}
64
+ block_el = etree.SubElement(parent, _a("TextBlock"), attrib)
65
+
66
+ # Résolution du texte pour ce bloc
67
+ text = ""
68
+ block_confidence = confidence
69
+
70
+ if ocr_block:
71
+ text = (
72
+ ocr_block.get("text")
73
+ or ocr_block.get("diplomatic_text")
74
+ or ""
75
+ )
76
+ if not text and ocr_block.get("lines"):
77
+ text = " ".join(
78
+ ln.get("text", "") for ln in ocr_block["lines"] if ln.get("text")
79
+ )
80
+ block_confidence = float(ocr_block.get("confidence", confidence))
81
+ elif fallback_text:
82
+ text = fallback_text
83
+
84
+ if not text:
85
+ return # TextBlock vide — valide ALTO
86
+
87
+ x, y, w, h = region.bbox
88
+ line_el = etree.SubElement(
89
+ block_el,
90
+ _a("TextLine"),
91
+ {"ID": f"{region.id}_l1", "HPOS": str(x), "VPOS": str(y), "WIDTH": str(w), "HEIGHT": str(h)},
92
+ )
93
+ etree.SubElement(
94
+ line_el,
95
+ _a("String"),
96
+ {
97
+ "ID": f"{region.id}_l1_s1",
98
+ "CONTENT": text,
99
+ "HPOS": str(x),
100
+ "VPOS": str(y),
101
+ "WIDTH": str(w),
102
+ "HEIGHT": str(h),
103
+ "WC": f"{block_confidence:.4f}",
104
+ },
105
+ )
106
+
107
+
108
+ def generate_alto(master: PageMaster) -> str:
109
+ """Génère le XML ALTO v4 complet depuis un PageMaster.
110
+
111
+ Toutes les régions du layout sont validées avant la génération.
112
+ Un master.json avec une région invalide lève une ValueError explicite —
113
+ jamais d'ALTO partiel silencieux.
114
+
115
+ Args:
116
+ master: PageMaster Pydantic validé (source canonique, R02).
117
+
118
+ Returns:
119
+ Chaîne UTF-8 contenant le XML ALTO v4 (avec déclaration XML).
120
+
121
+ Raises:
122
+ ValueError: si une région du layout est invalide (bbox incorrecte, champ manquant).
123
+ """
124
+ # ── 1. Validation stricte de toutes les régions ─────────────────────────
125
+ raw_regions: list[dict] = (master.layout.get("regions") or [])
126
+ regions: list[Region] = []
127
+ for i, raw in enumerate(raw_regions):
128
+ try:
129
+ regions.append(Region.model_validate(raw))
130
+ except (ValidationError, Exception) as exc:
131
+ raise ValueError(
132
+ f"Région [{i}] invalide dans le layout de la page «{master.page_id}» : {exc}"
133
+ ) from exc
134
+
135
+ # ── 2. Index OCR par region_id ──────────────────────────────────────────
136
+ ocr: OCRResult | None = master.ocr
137
+ language = (ocr.language if ocr else "la") or "la"
138
+ global_confidence = ocr.confidence if ocr else 0.0
139
+ global_text = (ocr.diplomatic_text if ocr else "") or ""
140
+
141
+ ocr_by_region: dict[str, dict] = {}
142
+ if ocr:
143
+ for block in ocr.blocks:
144
+ rid = block.get("region_id")
145
+ if rid:
146
+ ocr_by_region[rid] = block
147
+
148
+ # Fallback : si aucun bloc OCR n'est référencé par region_id,
149
+ # le texte diplomatique global ira dans le premier TextBlock.
150
+ has_per_region_ocr = bool(ocr_by_region)
151
+ first_text_block_done = False
152
+
153
+ # ── 3. Construction de l'arbre XML ─────────────────────────────────────
154
+ nsmap = {None: _ALTO_NS, "xsi": _XSI_NS}
155
+ root = etree.Element(_a("alto"), nsmap=nsmap)
156
+ root.set(f"{{{_XSI_NS}}}schemaLocation", _SCHEMA_LOC)
157
+
158
+ # ── Description ────────────────────────────────────────────────────────
159
+ desc = etree.SubElement(root, _a("Description"))
160
+ etree.SubElement(desc, _a("MeasurementUnit")).text = "pixel"
161
+
162
+ src_info = etree.SubElement(desc, _a("sourceImageInformation"))
163
+ file_name = (
164
+ master.image.get("original_url")
165
+ or master.image.get("derivative_web")
166
+ or master.page_id
167
+ )
168
+ etree.SubElement(src_info, _a("fileName")).text = str(file_name)
169
+
170
+ if master.processing:
171
+ ocr_proc = etree.SubElement(desc, _a("OCRProcessing"), {"ID": "OCR_1"})
172
+ step = etree.SubElement(ocr_proc, _a("ocrProcessingStep"))
173
+ etree.SubElement(step, _a("processingDateTime")).text = (
174
+ master.processing.processed_at.isoformat()
175
+ )
176
+ software = etree.SubElement(step, _a("processingSoftware"))
177
+ etree.SubElement(software, _a("softwareCreator")).text = "Scriptorium AI"
178
+ etree.SubElement(software, _a("softwareName")).text = (
179
+ master.processing.model_display_name
180
+ )
181
+ etree.SubElement(software, _a("softwareVersion")).text = (
182
+ master.processing.prompt_version
183
+ )
184
+
185
+ # ── Layout ─────────────────────────────────────────────────────────────
186
+ layout_el = etree.SubElement(root, _a("Layout"))
187
+
188
+ width = int(master.image.get("width", 0))
189
+ height = int(master.image.get("height", 0))
190
+
191
+ page_id_safe = master.page_id.replace(" ", "_")
192
+ page_el = etree.SubElement(
193
+ layout_el,
194
+ _a("Page"),
195
+ {
196
+ "ID": f"P_{page_id_safe}",
197
+ "PHYSICAL_IMG_NR": str(master.sequence),
198
+ "WIDTH": str(width),
199
+ "HEIGHT": str(height),
200
+ },
201
+ )
202
+ print_space = etree.SubElement(
203
+ page_el,
204
+ _a("PrintSpace"),
205
+ {"HPOS": "0", "VPOS": "0", "WIDTH": str(width), "HEIGHT": str(height)},
206
+ )
207
+
208
+ # ── Régions ────────────────────────────────────────────────────────────
209
+ for region in regions:
210
+ if region.type in _TEXT_REGION_TYPES:
211
+ ocr_block = ocr_by_region.get(region.id)
212
+ fallback = ""
213
+ if not has_per_region_ocr and not first_text_block_done and global_text:
214
+ fallback = global_text
215
+ first_text_block_done = True
216
+
217
+ _build_text_block(
218
+ print_space,
219
+ region,
220
+ ocr_block=ocr_block,
221
+ fallback_text=fallback,
222
+ language=language,
223
+ confidence=global_confidence,
224
+ )
225
+
226
+ elif region.type in _ILLUSTRATION_TYPES:
227
+ etree.SubElement(
228
+ print_space,
229
+ _a("Illustration"),
230
+ {
231
+ "ID": region.id,
232
+ "TYPE": region.type.value,
233
+ **_bbox_attrib(region),
234
+ },
235
+ )
236
+
237
+ else: # _COMPOSED_TYPES (other)
238
+ etree.SubElement(
239
+ print_space,
240
+ _a("ComposedBlock"),
241
+ {"ID": region.id, **_bbox_attrib(region)},
242
+ )
243
+
244
+ logger.info(
245
+ "ALTO généré",
246
+ extra={
247
+ "page_id": master.page_id,
248
+ "regions": len(regions),
249
+ },
250
+ )
251
+
252
+ # ── 4. Sérialisation ───────────────────────────────────────────────────
253
+ return etree.tostring(
254
+ root,
255
+ xml_declaration=True,
256
+ encoding="UTF-8",
257
+ pretty_print=True,
258
+ ).decode("utf-8")
259
+
260
+
261
+ def write_alto(alto_xml: str, output_path: Path) -> None:
262
+ """Écrit le XML ALTO dans le fichier de sortie.
263
+
264
+ Crée les dossiers parents si nécessaire.
265
+
266
+ Args:
267
+ alto_xml: chaîne XML retournée par generate_alto().
268
+ output_path: chemin de sortie (typiquement .../pages/{folio}/alto.xml).
269
+ """
270
+ output_path.parent.mkdir(parents=True, exist_ok=True)
271
+ output_path.write_text(alto_xml, encoding="utf-8")
272
+ logger.info("alto.xml écrit", extra={"path": str(output_path)})
backend/tests/test_export_alto.py ADDED
@@ -0,0 +1,650 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Tests du générateur ALTO v4 (Sprint 3 — Session A).
3
+
4
+ Vérifie :
5
+ - XML produit est valide (parseable par lxml)
6
+ - Correspondance exacte bbox → HPOS/VPOS/WIDTH/HEIGHT
7
+ - text_block / margin / rubric → TextBlock
8
+ - miniature / decorated_initial → Illustration
9
+ - other → ComposedBlock
10
+ - Texte OCR présent dans TextBlock quand disponible
11
+ - Fallback diplomatic_text dans le premier TextBlock si aucun bloc OCR par région
12
+ - Master sans régions → ALTO valide avec PrintSpace vide
13
+ - Région invalide → ValueError explicite (jamais ALTO partiel)
14
+ - Dimensions de page (WIDTH/HEIGHT) issues de master.image
15
+ - OCRProcessing présent si master.processing, absent sinon
16
+ - Attribut TYPE sur Illustration (valeur du RegionType)
17
+ - Page ID contient le page_id du master
18
+ """
19
+ # 1. stdlib
20
+ import json
21
+ from datetime import datetime, timezone
22
+ from pathlib import Path
23
+
24
+ # 2. third-party
25
+ import pytest
26
+ from lxml import etree
27
+
28
+ # 3. local
29
+ from app.schemas.page_master import EditorialInfo, EditorialStatus, OCRResult, PageMaster, ProcessingInfo
30
+ from app.services.export.alto import generate_alto, write_alto
31
+
32
+ # ── Namespace ALTO v4 ─────────────────────────────────────────────────────
33
+ _ALTO_NS = "http://www.loc.gov/standards/alto/ns-v4#"
34
+ _A = f"{{{_ALTO_NS}}}"
35
+
36
+
37
+ # ---------------------------------------------------------------------------
38
+ # Helpers / Fixtures
39
+ # ---------------------------------------------------------------------------
40
+
41
+ def _make_master(
42
+ page_id: str = "test-ms-0001r",
43
+ sequence: int = 1,
44
+ regions: list | None = None,
45
+ ocr: OCRResult | None = None,
46
+ width: int = 1500,
47
+ height: int = 2000,
48
+ with_processing: bool = False,
49
+ ) -> PageMaster:
50
+ if regions is None:
51
+ regions = []
52
+ processing = None
53
+ if with_processing:
54
+ processing = ProcessingInfo(
55
+ model_id="gemini-2.0-flash",
56
+ model_display_name="Gemini 2.0 Flash",
57
+ prompt_version="prompts/medieval-illuminated/primary_v1.txt",
58
+ raw_response_path="/data/gemini_raw.json",
59
+ processed_at=datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc),
60
+ )
61
+ return PageMaster(
62
+ page_id=page_id,
63
+ corpus_profile="medieval-illuminated",
64
+ manuscript_id="ms-test",
65
+ folio_label="0001r",
66
+ sequence=sequence,
67
+ image={
68
+ "original_url": "https://example.com/img.jpg",
69
+ "derivative_web": "/data/deriv.jpg",
70
+ "thumbnail": "/data/thumb.jpg",
71
+ "width": width,
72
+ "height": height,
73
+ },
74
+ layout={"regions": regions},
75
+ ocr=ocr,
76
+ processing=processing,
77
+ editorial=EditorialInfo(status=EditorialStatus.MACHINE_DRAFT),
78
+ )
79
+
80
+
81
+ def _parse(xml_str: str) -> etree._Element:
82
+ """Parse une chaîne XML et retourne la racine."""
83
+ return etree.fromstring(xml_str.encode("utf-8"))
84
+
85
+
86
+ def _xpath(root: etree._Element, path: str) -> list:
87
+ return root.xpath(path, namespaces={"a": _ALTO_NS})
88
+
89
+
90
+ def _one(root: etree._Element, path: str) -> etree._Element:
91
+ results = _xpath(root, path)
92
+ assert len(results) == 1, f"Expected 1 match for {path!r}, got {len(results)}"
93
+ return results[0]
94
+
95
+
96
+ # ---------------------------------------------------------------------------
97
+ # Synthetic master.json fixtures (équivalents aux 3 master.json du Sprint 2)
98
+ # ---------------------------------------------------------------------------
99
+
100
+ @pytest.fixture
101
+ def master_text_only():
102
+ """Master simulant une page texte (seul TextBlock, OCR global)."""
103
+ return _make_master(
104
+ page_id="beatus-hr-0001r",
105
+ sequence=1,
106
+ regions=[
107
+ {"id": "r1", "type": "text_block", "bbox": [50, 100, 1400, 1800], "confidence": 0.92},
108
+ ],
109
+ ocr=OCRResult(
110
+ diplomatic_text="Incipit explanatio beati Ieronimi",
111
+ language="la",
112
+ confidence=0.92,
113
+ ),
114
+ width=1500,
115
+ height=2000,
116
+ with_processing=True,
117
+ )
118
+
119
+
120
+ @pytest.fixture
121
+ def master_mixed_regions():
122
+ """Master simulant un folio enluminé (texte + miniature + initial décoré)."""
123
+ return _make_master(
124
+ page_id="beatus-br-0013r",
125
+ sequence=13,
126
+ regions=[
127
+ {"id": "r1", "type": "miniature", "bbox": [0, 0, 1500, 800], "confidence": 0.95},
128
+ {"id": "r2", "type": "decorated_initial", "bbox": [50, 820, 200, 200], "confidence": 0.88},
129
+ {"id": "r3", "type": "text_block", "bbox": [260, 820, 1200, 200], "confidence": 0.90},
130
+ {"id": "r4", "type": "rubric", "bbox": [50, 1040, 1400, 80], "confidence": 0.85},
131
+ {"id": "r5", "type": "margin", "bbox": [0, 100, 45, 1800], "confidence": 0.70},
132
+ ],
133
+ ocr=OCRResult(
134
+ diplomatic_text="Sequitur de bestia",
135
+ language="la",
136
+ confidence=0.90,
137
+ ),
138
+ width=1500,
139
+ height=2000,
140
+ )
141
+
142
+
143
+ @pytest.fixture
144
+ def master_with_per_region_ocr():
145
+ """Master avec OCR indexé par region_id dans ocr.blocks."""
146
+ return _make_master(
147
+ page_id="chroniques-f016",
148
+ sequence=16,
149
+ regions=[
150
+ {"id": "r1", "type": "text_block", "bbox": [100, 100, 600, 400], "confidence": 0.91},
151
+ {"id": "r2", "type": "text_block", "bbox": [100, 520, 600, 300], "confidence": 0.87},
152
+ {"id": "r3", "type": "miniature", "bbox": [720, 100, 700, 800], "confidence": 0.96},
153
+ ],
154
+ ocr=OCRResult(
155
+ diplomatic_text="[global fallback]",
156
+ blocks=[
157
+ {"region_id": "r1", "text": "Cy commence le prologue", "confidence": 0.91},
158
+ {"region_id": "r2", "text": "Des grandes chroniques de France", "confidence": 0.87},
159
+ ],
160
+ language="fr",
161
+ confidence=0.89,
162
+ ),
163
+ width=1500,
164
+ height=2000,
165
+ )
166
+
167
+
168
+ # ---------------------------------------------------------------------------
169
+ # Tests — validité XML
170
+ # ---------------------------------------------------------------------------
171
+
172
+ def test_generate_alto_returns_string(master_text_only):
173
+ result = generate_alto(master_text_only)
174
+ assert isinstance(result, str)
175
+
176
+
177
+ def test_generate_alto_valid_xml(master_text_only):
178
+ """Le XML produit doit être parseable par lxml sans erreur."""
179
+ xml_str = generate_alto(master_text_only)
180
+ root = _parse(xml_str)
181
+ assert root is not None
182
+
183
+
184
+ def test_generate_alto_valid_xml_mixed(master_mixed_regions):
185
+ xml_str = generate_alto(master_mixed_regions)
186
+ root = _parse(xml_str)
187
+ assert root is not None
188
+
189
+
190
+ def test_generate_alto_valid_xml_per_region_ocr(master_with_per_region_ocr):
191
+ xml_str = generate_alto(master_with_per_region_ocr)
192
+ root = _parse(xml_str)
193
+ assert root is not None
194
+
195
+
196
+ def test_generate_alto_xml_declaration(master_text_only):
197
+ """Le XML doit commencer par la déclaration XML."""
198
+ xml_str = generate_alto(master_text_only)
199
+ assert xml_str.startswith("<?xml")
200
+
201
+
202
+ def test_generate_alto_alto_namespace(master_text_only):
203
+ """L'élément racine doit utiliser le namespace ALTO v4."""
204
+ root = _parse(generate_alto(master_text_only))
205
+ assert root.tag == f"{{{_ALTO_NS}}}alto"
206
+
207
+
208
+ def test_generate_alto_empty_regions():
209
+ """Un master sans régions produit un ALTO valide avec PrintSpace vide."""
210
+ master = _make_master(regions=[])
211
+ xml_str = generate_alto(master)
212
+ root = _parse(xml_str)
213
+ ps = _one(root, "//a:PrintSpace")
214
+ assert len(list(ps)) == 0 # aucun enfant
215
+
216
+
217
+ # ---------------------------------------------------------------------------
218
+ # Tests — bbox HPOS/VPOS/WIDTH/HEIGHT (R03 correspondance directe)
219
+ # ---------------------------------------------------------------------------
220
+
221
+ def test_text_block_bbox_exact(master_text_only):
222
+ """HPOS/VPOS/WIDTH/HEIGHT du TextBlock correspondent exactement à bbox [x,y,w,h]."""
223
+ root = _parse(generate_alto(master_text_only))
224
+ tb = _one(root, "//a:TextBlock[@ID='r1']")
225
+ assert tb.get("HPOS") == "50"
226
+ assert tb.get("VPOS") == "100"
227
+ assert tb.get("WIDTH") == "1400"
228
+ assert tb.get("HEIGHT") == "1800"
229
+
230
+
231
+ def test_illustration_bbox_exact(master_mixed_regions):
232
+ """HPOS/VPOS/WIDTH/HEIGHT de l'Illustration correspondent exactement à bbox."""
233
+ root = _parse(generate_alto(master_mixed_regions))
234
+ ill = _one(root, "//a:Illustration[@ID='r1']")
235
+ assert ill.get("HPOS") == "0"
236
+ assert ill.get("VPOS") == "0"
237
+ assert ill.get("WIDTH") == "1500"
238
+ assert ill.get("HEIGHT") == "800"
239
+
240
+
241
+ def test_decorated_initial_bbox_exact(master_mixed_regions):
242
+ root = _parse(generate_alto(master_mixed_regions))
243
+ ill = _one(root, "//a:Illustration[@ID='r2']")
244
+ assert ill.get("HPOS") == "50"
245
+ assert ill.get("VPOS") == "820"
246
+ assert ill.get("WIDTH") == "200"
247
+ assert ill.get("HEIGHT") == "200"
248
+
249
+
250
+ def test_multiple_text_blocks_bbox(master_with_per_region_ocr):
251
+ root = _parse(generate_alto(master_with_per_region_ocr))
252
+ tb1 = _one(root, "//a:TextBlock[@ID='r1']")
253
+ tb2 = _one(root, "//a:TextBlock[@ID='r2']")
254
+
255
+ assert (tb1.get("HPOS"), tb1.get("VPOS"), tb1.get("WIDTH"), tb1.get("HEIGHT")) == (
256
+ "100", "100", "600", "400"
257
+ )
258
+ assert (tb2.get("HPOS"), tb2.get("VPOS"), tb2.get("WIDTH"), tb2.get("HEIGHT")) == (
259
+ "100", "520", "600", "300"
260
+ )
261
+
262
+
263
+ # ---------------------------------------------------------------------------
264
+ # Tests — mapping RegionType → élément ALTO
265
+ # ---------------------------------------------------------------------------
266
+
267
+ def test_text_block_produces_TextBlock(master_text_only):
268
+ root = _parse(generate_alto(master_text_only))
269
+ assert len(_xpath(root, "//a:TextBlock")) == 1
270
+ assert len(_xpath(root, "//a:Illustration")) == 0
271
+
272
+
273
+ def test_miniature_produces_Illustration(master_mixed_regions):
274
+ root = _parse(generate_alto(master_mixed_regions))
275
+ ill = _one(root, "//a:Illustration[@ID='r1']")
276
+ assert ill.get("TYPE") == "miniature"
277
+
278
+
279
+ def test_decorated_initial_produces_Illustration(master_mixed_regions):
280
+ root = _parse(generate_alto(master_mixed_regions))
281
+ ill = _one(root, "//a:Illustration[@ID='r2']")
282
+ assert ill.get("TYPE") == "decorated_initial"
283
+
284
+
285
+ def test_rubric_produces_TextBlock(master_mixed_regions):
286
+ root = _parse(generate_alto(master_mixed_regions))
287
+ # r4 est rubric → doit être un TextBlock, pas une Illustration
288
+ tb = _one(root, "//a:TextBlock[@ID='r4']")
289
+ assert tb is not None
290
+ illustrations = _xpath(root, "//a:Illustration[@ID='r4']")
291
+ assert len(illustrations) == 0
292
+
293
+
294
+ def test_margin_produces_TextBlock(master_mixed_regions):
295
+ root = _parse(generate_alto(master_mixed_regions))
296
+ tb = _one(root, "//a:TextBlock[@ID='r5']")
297
+ assert tb is not None
298
+
299
+
300
+ def test_other_region_produces_ComposedBlock():
301
+ master = _make_master(
302
+ regions=[{"id": "r1", "type": "other", "bbox": [10, 10, 100, 100], "confidence": 0.5}]
303
+ )
304
+ root = _parse(generate_alto(master))
305
+ cb = _one(root, "//a:ComposedBlock[@ID='r1']")
306
+ assert cb.get("HPOS") == "10"
307
+ assert cb.get("VPOS") == "10"
308
+ assert cb.get("WIDTH") == "100"
309
+ assert cb.get("HEIGHT") == "100"
310
+ assert len(_xpath(root, "//a:TextBlock")) == 0
311
+ assert len(_xpath(root, "//a:Illustration")) == 0
312
+
313
+
314
+ def test_mixed_regions_counts(master_mixed_regions):
315
+ """Folio enluminé : 3 TextBlock (text_block + rubric + margin) + 2 Illustration."""
316
+ root = _parse(generate_alto(master_mixed_regions))
317
+ assert len(_xpath(root, "//a:TextBlock")) == 3
318
+ assert len(_xpath(root, "//a:Illustration")) == 2
319
+
320
+
321
+ # ---------------------------------------------------------------------------
322
+ # Tests — contenu OCR dans les TextBlocks
323
+ # ---------------------------------------------------------------------------
324
+
325
+ def test_diplomatic_text_in_first_text_block(master_text_only):
326
+ """Le diplomatic_text global va dans le premier TextBlock si pas de per-region OCR."""
327
+ root = _parse(generate_alto(master_text_only))
328
+ strings = _xpath(root, "//a:TextBlock[@ID='r1']/a:TextLine/a:String")
329
+ assert len(strings) == 1
330
+ assert strings[0].get("CONTENT") == "Incipit explanatio beati Ieronimi"
331
+
332
+
333
+ def test_per_region_ocr_blocks_used(master_with_per_region_ocr):
334
+ """Quand ocr.blocks contient des region_id, chaque TextBlock a son propre texte."""
335
+ root = _parse(generate_alto(master_with_per_region_ocr))
336
+
337
+ s1 = _one(root, "//a:TextBlock[@ID='r1']/a:TextLine/a:String")
338
+ s2 = _one(root, "//a:TextBlock[@ID='r2']/a:TextLine/a:String")
339
+
340
+ assert s1.get("CONTENT") == "Cy commence le prologue"
341
+ assert s2.get("CONTENT") == "Des grandes chroniques de France"
342
+
343
+
344
+ def test_per_region_ocr_no_fallback_in_second_block(master_with_per_region_ocr):
345
+ """Le fallback global N'est PAS répété dans d'autres blocs quand per-region OCR existe."""
346
+ root = _parse(generate_alto(master_with_per_region_ocr))
347
+ strings = _xpath(root, "//a:String[@CONTENT='[global fallback]']")
348
+ assert len(strings) == 0
349
+
350
+
351
+ def test_no_ocr_text_block_is_empty():
352
+ """TextBlock sans OCR disponible reste vide (valide ALTO)."""
353
+ master = _make_master(
354
+ regions=[{"id": "r1", "type": "text_block", "bbox": [0, 0, 100, 100], "confidence": 0.8}],
355
+ ocr=None,
356
+ )
357
+ root = _parse(generate_alto(master))
358
+ tb = _one(root, "//a:TextBlock[@ID='r1']")
359
+ assert len(list(tb)) == 0 # pas d'enfants TextLine
360
+
361
+
362
+ def test_text_content_not_in_illustration():
363
+ """Les Illustrations ne doivent pas contenir de TextLine."""
364
+ master = _make_master(
365
+ regions=[{"id": "r1", "type": "miniature", "bbox": [0, 0, 500, 500], "confidence": 0.9}],
366
+ ocr=OCRResult(diplomatic_text="Some text that shouldn't be in illustration"),
367
+ )
368
+ root = _parse(generate_alto(master))
369
+ assert len(_xpath(root, "//a:Illustration/a:TextLine")) == 0
370
+ assert len(_xpath(root, "//a:Illustration/a:String")) == 0
371
+
372
+
373
+ def test_string_wc_attribute_present(master_text_only):
374
+ """L'attribut WC (word confidence) est présent sur les éléments String."""
375
+ root = _parse(generate_alto(master_text_only))
376
+ strings = _xpath(root, "//a:String")
377
+ for s in strings:
378
+ assert s.get("WC") is not None
379
+ wc = float(s.get("WC"))
380
+ assert 0.0 <= wc <= 1.0
381
+
382
+
383
+ def test_string_bbox_matches_region_bbox(master_text_only):
384
+ """Le String hérite des coordonnées bbox de sa région."""
385
+ root = _parse(generate_alto(master_text_only))
386
+ s = _one(root, "//a:TextBlock[@ID='r1']/a:TextLine/a:String")
387
+ assert s.get("HPOS") == "50"
388
+ assert s.get("VPOS") == "100"
389
+ assert s.get("WIDTH") == "1400"
390
+ assert s.get("HEIGHT") == "1800"
391
+
392
+
393
+ # ---------------------------------------------------------------------------
394
+ # Tests — dimensions de page
395
+ # ---------------------------------------------------------------------------
396
+
397
+ def test_page_width_height_from_image(master_text_only):
398
+ root = _parse(generate_alto(master_text_only))
399
+ page = _one(root, "//a:Page")
400
+ assert page.get("WIDTH") == "1500"
401
+ assert page.get("HEIGHT") == "2000"
402
+
403
+
404
+ def test_print_space_dimensions_match_page(master_text_only):
405
+ root = _parse(generate_alto(master_text_only))
406
+ ps = _one(root, "//a:PrintSpace")
407
+ assert ps.get("HPOS") == "0"
408
+ assert ps.get("VPOS") == "0"
409
+ assert ps.get("WIDTH") == "1500"
410
+ assert ps.get("HEIGHT") == "2000"
411
+
412
+
413
+ def test_page_physical_img_nr_is_sequence(master_mixed_regions):
414
+ root = _parse(generate_alto(master_mixed_regions))
415
+ page = _one(root, "//a:Page")
416
+ assert page.get("PHYSICAL_IMG_NR") == "13"
417
+
418
+
419
+ def test_page_id_contains_page_id(master_text_only):
420
+ root = _parse(generate_alto(master_text_only))
421
+ page = _one(root, "//a:Page")
422
+ assert "beatus-hr-0001r" in page.get("ID", "")
423
+
424
+
425
+ # ---------------------------------------------------------------------------
426
+ # Tests — OCRProcessing (metadata de traitement)
427
+ # ---------------------------------------------------------------------------
428
+
429
+ def test_ocr_processing_present_when_processing_info(master_text_only):
430
+ """OCRProcessing est présent si master.processing est défini."""
431
+ root = _parse(generate_alto(master_text_only))
432
+ ocr_proc = _one(root, "//a:OCRProcessing")
433
+ assert ocr_proc is not None
434
+
435
+
436
+ def test_ocr_processing_model_name(master_text_only):
437
+ root = _parse(generate_alto(master_text_only))
438
+ name = _one(root, "//a:processingSoftware/a:softwareName")
439
+ assert name.text == "Gemini 2.0 Flash"
440
+
441
+
442
+ def test_ocr_processing_prompt_version(master_text_only):
443
+ root = _parse(generate_alto(master_text_only))
444
+ version = _one(root, "//a:processingSoftware/a:softwareVersion")
445
+ assert "primary_v1.txt" in version.text
446
+
447
+
448
+ def test_ocr_processing_datetime(master_text_only):
449
+ root = _parse(generate_alto(master_text_only))
450
+ dt = _one(root, "//a:processingDateTime")
451
+ assert "2024-06-15" in dt.text
452
+
453
+
454
+ def test_ocr_processing_absent_without_processing_info():
455
+ """OCRProcessing est absent si master.processing est None."""
456
+ master = _make_master(with_processing=False)
457
+ root = _parse(generate_alto(master))
458
+ assert len(_xpath(root, "//a:OCRProcessing")) == 0
459
+
460
+
461
+ def test_software_creator_is_scriptorium_ai(master_text_only):
462
+ root = _parse(generate_alto(master_text_only))
463
+ creator = _one(root, "//a:processingSoftware/a:softwareCreator")
464
+ assert creator.text == "Scriptorium AI"
465
+
466
+
467
+ # ---------------------------------------------------------------------------
468
+ # Tests — source image
469
+ # ---------------------------------------------------------------------------
470
+
471
+ def test_source_filename_from_original_url(master_text_only):
472
+ root = _parse(generate_alto(master_text_only))
473
+ fn = _one(root, "//a:fileName")
474
+ assert fn.text == "https://example.com/img.jpg"
475
+
476
+
477
+ # ---------------------------------------------------------------------------
478
+ # Tests — erreur explicite sur region invalide
479
+ # ---------------------------------------------------------------------------
480
+
481
+ def test_invalid_region_raises_value_error():
482
+ """Une région avec bbox invalide dans le layout lève ValueError (jamais ALTO partiel)."""
483
+ master = _make_master(
484
+ regions=[
485
+ {"id": "r1", "type": "text_block", "bbox": [0, 0, 0, 100], "confidence": 0.9},
486
+ ]
487
+ )
488
+ with pytest.raises(ValueError, match="Région"):
489
+ generate_alto(master)
490
+
491
+
492
+ def test_invalid_region_negative_bbox_raises():
493
+ master = _make_master(
494
+ regions=[
495
+ {"id": "r1", "type": "text_block", "bbox": [-5, 0, 100, 100], "confidence": 0.9},
496
+ ]
497
+ )
498
+ with pytest.raises(ValueError, match="Région"):
499
+ generate_alto(master)
500
+
501
+
502
+ def test_invalid_region_missing_field_raises():
503
+ """Une région sans champ 'type' lève ValueError."""
504
+ master = _make_master(
505
+ regions=[
506
+ {"id": "r1", "bbox": [0, 0, 100, 100], "confidence": 0.9}, # manque 'type'
507
+ ]
508
+ )
509
+ with pytest.raises(ValueError, match="Région"):
510
+ generate_alto(master)
511
+
512
+
513
+ def test_valid_region_after_nothing_invalid_no_partial_output():
514
+ """Si la première région est invalide, aucun ALTO n'est produit (pas de sortie partielle)."""
515
+ master = _make_master(
516
+ regions=[
517
+ {"id": "r_bad", "type": "text_block", "bbox": [0, 0, -1, 100], "confidence": 0.9},
518
+ {"id": "r_good", "type": "miniature", "bbox": [0, 0, 100, 100], "confidence": 0.9},
519
+ ]
520
+ )
521
+ with pytest.raises(ValueError):
522
+ generate_alto(master)
523
+
524
+
525
+ # ---------------------------------------------------------------------------
526
+ # Tests — write_alto
527
+ # ---------------------------------------------------------------------------
528
+
529
+ def test_write_alto_creates_file(tmp_path):
530
+ master = _make_master()
531
+ xml_str = generate_alto(master)
532
+ out = tmp_path / "alto.xml"
533
+ write_alto(xml_str, out)
534
+ assert out.exists()
535
+
536
+
537
+ def test_write_alto_creates_parent_dirs(tmp_path):
538
+ master = _make_master()
539
+ xml_str = generate_alto(master)
540
+ out = tmp_path / "data" / "corpora" / "test" / "pages" / "0001r" / "alto.xml"
541
+ write_alto(xml_str, out)
542
+ assert out.exists()
543
+
544
+
545
+ def test_write_alto_content_is_valid_xml(tmp_path):
546
+ master = _make_master(
547
+ regions=[{"id": "r1", "type": "text_block", "bbox": [0, 0, 100, 100], "confidence": 0.8}]
548
+ )
549
+ xml_str = generate_alto(master)
550
+ out = tmp_path / "alto.xml"
551
+ write_alto(xml_str, out)
552
+ root = etree.parse(str(out)).getroot()
553
+ assert root.tag == f"{{{_ALTO_NS}}}alto"
554
+
555
+
556
+ # ---------------------------------------------------------------------------
557
+ # Tests — scénarios réalistes Sprint 2 (3 "master.json" simulés)
558
+ # ---------------------------------------------------------------------------
559
+
560
+ def test_beatus_hr_folio_alto():
561
+ """Simule le folio Beatus HR : page texte dense avec OCR global."""
562
+ master = _make_master(
563
+ page_id="beatus-lat8878-hr-f233",
564
+ sequence=233,
565
+ regions=[
566
+ {"id": "r1", "type": "text_block", "bbox": [42, 58, 1410, 1880], "confidence": 0.93},
567
+ {"id": "r2", "type": "rubric", "bbox": [42, 58, 400, 40], "confidence": 0.88},
568
+ {"id": "r3", "type": "margin", "bbox": [1460, 58, 40, 1880], "confidence": 0.60},
569
+ ],
570
+ ocr=OCRResult(
571
+ diplomatic_text="Et post hec uidit angelum descendentem de celo",
572
+ language="la",
573
+ confidence=0.93,
574
+ ),
575
+ width=1500,
576
+ height=2000,
577
+ with_processing=True,
578
+ )
579
+ root = _parse(generate_alto(master))
580
+ assert len(_xpath(root, "//a:TextBlock")) == 3
581
+ assert len(_xpath(root, "//a:Illustration")) == 0
582
+ # OCR dans le premier TextBlock
583
+ strings = _xpath(root, "//a:String")
584
+ assert any("angelum" in (s.get("CONTENT") or "") for s in strings)
585
+
586
+
587
+ def test_beatus_br_folio_alto():
588
+ """Simule le folio Beatus BR : miniature + texte."""
589
+ master = _make_master(
590
+ page_id="beatus-lat8878-br-f233",
591
+ sequence=233,
592
+ regions=[
593
+ {"id": "r1", "type": "miniature", "bbox": [0, 0, 1500, 900], "confidence": 0.97},
594
+ {"id": "r2", "type": "decorated_initial", "bbox": [42, 920, 120, 120], "confidence": 0.85},
595
+ {"id": "r3", "type": "text_block", "bbox": [180, 920, 1280, 120], "confidence": 0.91},
596
+ ],
597
+ ocr=OCRResult(
598
+ diplomatic_text="Incipit liber beati",
599
+ language="la",
600
+ confidence=0.91,
601
+ ),
602
+ width=1500,
603
+ height=2000,
604
+ )
605
+ root = _parse(generate_alto(master))
606
+ # Miniature + initial → Illustration
607
+ assert len(_xpath(root, "//a:Illustration")) == 2
608
+ # Un TextBlock
609
+ assert len(_xpath(root, "//a:TextBlock")) == 1
610
+ # Les Illustrations ne sont pas des TextBlocks
611
+ ill_ids = {el.get("ID") for el in _xpath(root, "//a:Illustration")}
612
+ tb_ids = {el.get("ID") for el in _xpath(root, "//a:TextBlock")}
613
+ assert ill_ids.isdisjoint(tb_ids)
614
+
615
+
616
+ def test_grandes_chroniques_folio_alto():
617
+ """Simule le folio Grandes Chroniques : texte en français, OCR par région."""
618
+ master = _make_master(
619
+ page_id="chroniques-btv1b84472995-f16",
620
+ sequence=16,
621
+ regions=[
622
+ {"id": "r1", "type": "miniature", "bbox": [20, 20, 700, 600], "confidence": 0.96},
623
+ {"id": "r2", "type": "text_block", "bbox": [740, 20, 720, 600], "confidence": 0.89},
624
+ {"id": "r3", "type": "text_block", "bbox": [20, 640, 1440, 1300], "confidence": 0.91},
625
+ ],
626
+ ocr=OCRResult(
627
+ diplomatic_text="",
628
+ blocks=[
629
+ {"region_id": "r2", "text": "Ci commence le prolog", "confidence": 0.89},
630
+ {"region_id": "r3", "text": "Des roys de france", "confidence": 0.91},
631
+ ],
632
+ language="fr",
633
+ confidence=0.90,
634
+ ),
635
+ width=1460,
636
+ height=1960,
637
+ )
638
+ root = _parse(generate_alto(master))
639
+ assert len(_xpath(root, "//a:Illustration")) == 1
640
+ assert len(_xpath(root, "//a:TextBlock")) == 2
641
+
642
+ s2 = _one(root, "//a:TextBlock[@ID='r2']/a:TextLine/a:String")
643
+ s3 = _one(root, "//a:TextBlock[@ID='r3']/a:TextLine/a:String")
644
+ assert s2.get("CONTENT") == "Ci commence le prolog"
645
+ assert s3.get("CONTENT") == "Des roys de france"
646
+
647
+ # Dimensions de page correctes
648
+ page = _one(root, "//a:Page")
649
+ assert page.get("WIDTH") == "1460"
650
+ assert page.get("HEIGHT") == "1960"