scripts/extract_layout.py

"""Re-extraction layout-fidele du manuel PCP, avec correction des glyphes Symbol.

Pourquoi : l'extraction v1 (get_text("text") + normalisation ASCII) perdait les
delimiteurs des boites de syntaxe. Diagnostic 2026-06-10 : ces delimiteurs sont
des glyphes de la police Symbol que PyMuPDF restitue en codepoints prives
(U+F000 + code Symbol) ; le normaliseur les supprimait ensuite. Les < > = [ ]
et les grandes accolades (alternatives empilees) sont donc RECUPERABLES
deterministiquement en mappant ces codes vers leur equivalent texte.

Sortie : pcp/data/text_layout/<doc>.txt — texte aligne en colonnes (style
pdftotext -layout) avec les bons caracteres, marqueurs [[page N | label]].
Sert de source de verite pour la reparation des champs syntax du catalogue
et la construction de la KB (docs/PLAN.md section 4).

Usage : python scripts/extract_layout.py [--doc chp02a.langage] [--src DIR]
"""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

import fitz  # PyMuPDF

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_SRC = ROOT.parent / "Documentation"
OUT_DIR = ROOT / "pcp" / "data" / "text_layout"

# Codes Adobe Symbol -> texte. Les pieces d'accolades/crochets multi-lignes
# (alternatives empilees des boites de syntaxe) sont rabattues sur { | } [ ] ( )
# ligne par ligne : une pile verticale d'accolades devient une colonne de { et }.
SYMBOL_MAP = {
    0x22: "∀", 0x24: "∃", 0x2A: "*", 0x2D: "-", 0x40: "≅",
    0xA3: "<=", 0xB3: ">=", 0xB1: "+/-", 0xB9: "!=", 0xB4: "x", 0xB8: "/",
    0xD6: "√", 0xA5: "inf", 0xAE: "->", 0xAC: "<-",
    # lettres grecques (minuscules puis majuscules usuelles)
    0x61: "α", 0x62: "β", 0x67: "γ", 0x64: "δ",
    0x65: "ε", 0x71: "θ", 0x6C: "λ", 0x6D: "μ",
    0x6E: "ν", 0x70: "π", 0x72: "ρ", 0x73: "σ",
    0x74: "τ", 0x66: "φ", 0x77: "ω", 0x68: "η",
    0x44: "Δ", 0x46: "Φ", 0x47: "Γ", 0x4C: "Λ",
    0x50: "Π", 0x51: "Θ", 0x53: "Σ", 0x57: "Ω",
    # pieces de parentheses / crochets / accolades multi-lignes
    0xE6: "(", 0xE7: "(", 0xE8: "(", 0xF6: ")", 0xF7: ")", 0xF8: ")",
    0xE9: "[", 0xEA: "[", 0xEB: "[", 0xF9: "]", 0xFA: "]", 0xFB: "]",
    0xEC: "{", 0xED: "{", 0xEE: "{", 0xEF: "|",
    0xFC: "}", 0xFD: "}", 0xFE: "}",
}

DINGBAT_FONTS = ("Wingdings", "ZapfDingbats", "MT-Extra")

# Largeur de colonne pour l'alignement (pt). Le corps du manuel est en 10-11 pt
# Times/Courier ; 5.2 pt par caractere donne un alignement stable sans etaler.
CHAR_W = 5.2
LINE_TOL = 3.0  # tolerance (pt) de regroupement vertical des spans en lignes


def _decode_pua(ch: str, font: str) -> str:
    code = ord(ch) - 0xF000
    if any(d in font for d in DINGBAT_FONTS):
        return "-"  # puces
    if "Symbol" in font and code in SYMBOL_MAP:
        return SYMBOL_MAP[code]
    if 0x20 <= code <= 0x7E:  # codes a glyphe quasi identique a l'ASCII
        return chr(code)
    return "?"


def _span_text(span: dict) -> str:
    font = span.get("font", "")
    out = []
    for ch in span["text"]:
        o = ord(ch)
        if 0xF000 <= o <= 0xF0FF:
            out.append(_decode_pua(ch, font))
        else:
            out.append(ch)
    return "".join(out)


def _page_lines(page: fitz.Page) -> list[str]:
    """Assemble les spans en lignes alignees en colonnes."""
    spans = []
    for block in page.get_text("dict")["blocks"]:
        for line in block.get("lines", []):
            for s in line["spans"]:
                txt = _span_text(s)
                if txt.strip() or txt:
                    x0, y0, x1, y1 = s["bbox"]
                    spans.append(((y0 + y1) / 2.0, x0, txt))
    if not spans:
        return []
    spans.sort(key=lambda t: (t[0], t[1]))
    # regroupement vertical
    rows: list[list[tuple[float, str]]] = []
    cur_y = None
    for yc, x0, txt in spans:
        if cur_y is None or yc - cur_y > LINE_TOL:
            rows.append([])
            cur_y = yc
        rows[-1].append((x0, txt))
    # placement en colonnes
    lines = []
    for row in rows:
        row.sort()
        buf = ""
        for x0, txt in row:
            col = int(round(x0 / CHAR_W))
            if col > len(buf):
                buf += " " * (col - len(buf))
            elif buf and not buf.endswith(" "):
                buf += " "
            buf += txt
        lines.append(buf.rstrip())
    return lines


def extract(pdf_path: Path, out_dir: Path) -> Path:
    doc = fitz.open(pdf_path)
    parts = []
    for pno in range(doc.page_count):
        page = doc[pno]
        lines = _page_lines(page)
        # libelle de page : derniere ligne courte du type "2-5"
        label = ""
        for ln in reversed(lines[-3:]):
            s = ln.strip()
            if 0 < len(s) <= 8 and "-" in s:
                label = s
                break
        parts.append(f"[[page {pno + 1} | {label}]]")
        parts.extend(lines)
        parts.append("")
    out = out_dir / (pdf_path.stem + ".txt")
    out.write_text("\n".join(parts), encoding="utf-8")
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--src", type=Path, default=DEFAULT_SRC)
    ap.add_argument("--doc", help="ne traiter que ce document (stem)")
    args = ap.parse_args()
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    pdfs = sorted(args.src.glob("*.pdf"))
    if args.doc:
        pdfs = [p for p in pdfs if p.stem == args.doc]
    if not pdfs:
        print(f"aucun PDF dans {args.src}", file=sys.stderr)
        return 1
    for pdf in pdfs:
        out = extract(pdf, OUT_DIR)
        n_lt = out.read_text(encoding="utf-8").count("<")
        print(f"{pdf.stem:40s} -> {out.name} ({n_lt} '<')")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())