File size: 13,832 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
#!/usr/bin/env python3
"""Analyze Unicode Limbu component classes and diacritic shaping behavior."""

from __future__ import annotations

import argparse
import csv
import itertools
import json
import random
import unicodedata
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Iterable

try:
    import uharfbuzz as hb
    from fontTools.ttLib import TTFont
    from PIL import Image, ImageDraw, ImageFont
except ImportError as exc:  # pragma: no cover - environment guard
    raise SystemExit(f"uharfbuzz, fontTools, and Pillow are required: {exc}") from exc

from ocrtech.errors import DataValidationError
from ocrtech.manifest import sha256_file


DEFAULT_FONT = Path("data/fonts/namdhinggo/Namdhinggo-3.100/Namdhinggo-Regular.ttf")
SCHEMA = "ocrtech.limbu-unicode-shaping-analysis.v1"


@dataclass(slots=True)
class ComponentClasses:
    bases: list[str]
    vowel_signs: list[str]
    subjoined: list[str]
    finals: list[str]
    signs: list[str]
    symbols: list[str]
    punctuation: list[str]
    digits: list[str]


@dataclass(slots=True)
class ShapeResult:
    family: str
    text: str
    codepoints: str
    names: str
    clean: bool
    visible: bool
    glyph_count: int
    has_notdef: bool
    has_dotted_circle: bool
    dark_pixels: int
    min_gray: int
    bbox_width: int
    bbox_height: int


def _assigned_limbu_chars() -> list[str]:
    chars: list[str] = []
    for cp in range(0x1900, 0x1950):
        ch = chr(cp)
        try:
            unicodedata.name(ch)
        except ValueError:
            continue
        chars.append(ch)
    return chars


def _classes() -> ComponentClasses:
    chars = _assigned_limbu_chars()
    bases = [ch for ch in chars if 0x1900 <= ord(ch) <= 0x191E]
    vowel_signs = [ch for ch in chars if 0x1920 <= ord(ch) <= 0x1928]
    subjoined = [ch for ch in chars if 0x1929 <= ord(ch) <= 0x192B]
    finals = [ch for ch in chars if 0x1930 <= ord(ch) <= 0x1938]
    signs = [ch for ch in chars if 0x1939 <= ord(ch) <= 0x193B]
    symbols = [ch for ch in chars if ord(ch) == 0x1940]
    punctuation = [ch for ch in chars if 0x1944 <= ord(ch) <= 0x1945]
    digits = [ch for ch in chars if 0x1946 <= ord(ch) <= 0x194F]
    return ComponentClasses(bases, vowel_signs, subjoined, finals, signs, symbols, punctuation, digits)


class LimbuShapeProbe:
    def __init__(self, font_path: Path, font_size: int, dark_threshold: int):
        if not font_path.is_file():
            raise DataValidationError(f"font does not exist: {font_path}")
        self.font_path = font_path
        self.font_size = font_size
        self.dark_threshold = dark_threshold
        self.font_data = font_path.read_bytes()
        self.hb_font = hb.Font(hb.Face(self.font_data))
        self.pil_font = ImageFont.truetype(str(font_path), font_size, layout_engine=ImageFont.Layout.RAQM)
        tt = TTFont(str(font_path))
        order = tt.getGlyphOrder()
        cmap = tt.getBestCmap() or {}
        self.bad_glyph_ids = {0}
        self.dotted_gid = None
        if 0x25CC in cmap:
            self.dotted_gid = order.index(cmap[0x25CC])
            self.bad_glyph_ids.add(self.dotted_gid)

    def shape(self, text: str) -> tuple[int, bool, bool]:
        buf = hb.Buffer()
        buf.add_str(text)
        buf.guess_segment_properties()
        hb.shape(self.hb_font, buf, {})
        glyph_ids = [info.codepoint for info in buf.glyph_infos]
        has_notdef = any(gid == 0 for gid in glyph_ids)
        has_dotted = self.dotted_gid is not None and any(gid == self.dotted_gid for gid in glyph_ids)
        return len(glyph_ids), has_notdef, has_dotted

    def render_stats(self, text: str) -> tuple[int, int, int, int]:
        probe = Image.new("RGB", (2048, 256), "white")
        draw = ImageDraw.Draw(probe)
        bbox = draw.textbbox((16, 32), text, font=self.pil_font)
        draw.text((16, 32), text, font=self.pil_font, fill=(0, 0, 0))
        gray = probe.convert("L")
        histogram = gray.histogram()
        dark_pixels = sum(histogram[: self.dark_threshold])
        min_gray = next((value for value, count in enumerate(histogram) if count), 255)
        width = max(0, bbox[2] - bbox[0])
        height = max(0, bbox[3] - bbox[1])
        return dark_pixels, min_gray, width, height

    def analyze(self, family: str, text: str) -> ShapeResult:
        glyph_count, has_notdef, has_dotted = self.shape(text)
        dark_pixels, min_gray, width, height = self.render_stats(text)
        visible = dark_pixels > 0 and height > 0 and width > 0
        clean = visible and not has_notdef and not has_dotted
        return ShapeResult(
            family=family,
            text=text,
            codepoints=" ".join(f"U+{ord(ch):04X}" for ch in text),
            names="; ".join(unicodedata.name(ch, "<no name>") for ch in text),
            clean=clean,
            visible=visible,
            glyph_count=glyph_count,
            has_notdef=has_notdef,
            has_dotted_circle=has_dotted,
            dark_pixels=dark_pixels,
            min_gray=min_gray,
            bbox_width=width,
            bbox_height=height,
        )


def _add_family(cases: list[tuple[str, str]], family: str, texts: Iterable[str]) -> None:
    for text in texts:
        cases.append((family, text))


def _build_cases(args: argparse.Namespace) -> list[tuple[str, str]]:
    c = _classes()
    cases: list[tuple[str, str]] = []
    _add_family(cases, "single_assigned", _assigned_limbu_chars())
    _add_family(cases, "base_vowel", (b + v for b in c.bases for v in c.vowel_signs))
    _add_family(cases, "base_subjoined", (b + s for b in c.bases for s in c.subjoined))
    _add_family(cases, "base_final", (b + f for b in c.bases for f in c.finals))
    _add_family(cases, "base_sign", (b + s for b in c.bases for s in c.signs))
    _add_family(cases, "base_vowel_final", (b + v + f for b in c.bases for v in c.vowel_signs for f in c.finals))
    _add_family(cases, "base_vowel_sign", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.signs))
    _add_family(cases, "base_final_sign", (b + f + s for b in c.bases for f in c.finals for s in c.signs))
    _add_family(
        cases,
        "base_vowel_final_sign",
        (b + v + f + s for b in c.bases for v in c.vowel_signs for f in c.finals for s in c.signs),
    )
    _add_family(cases, "base_subjoined_vowel", (b + s + v for b in c.bases for s in c.subjoined for v in c.vowel_signs))
    _add_family(cases, "base_vowel_subjoined", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.subjoined))
    _add_family(cases, "base_subjoined_final", (b + s + f for b in c.bases for s in c.subjoined for f in c.finals))
    _add_family(cases, "base_final_subjoined", (b + f + s for b in c.bases for f in c.finals for s in c.subjoined))
    _add_family(
        cases,
        "punct_symbol_digit_context",
        ("ᤁᤡ " + ch + " ᤕᤠ" for ch in c.symbols + c.punctuation + c.digits),
    )

    rng = random.Random(args.seed)
    components = c.vowel_signs + c.subjoined + c.finals + c.signs
    for index in range(args.random_order_samples):
        b = rng.choice(c.bases)
        chosen = [rng.choice(c.vowel_signs), rng.choice(c.subjoined), rng.choice(c.finals), rng.choice(c.signs)]
        rng.shuffle(chosen)
        cases.append(("random_component_order", b + "".join(chosen)))
        if index < args.random_canonical_samples:
            cases.append(
                (
                    "random_canonical_order",
                    b + rng.choice(c.vowel_signs) + rng.choice(c.subjoined) + rng.choice(c.finals) + rng.choice(c.signs),
                )
            )
    if args.limit and len(cases) > args.limit:
        rng.shuffle(cases)
        cases = cases[: args.limit]
    return cases


def _summarize(results: list[ShapeResult], font_path: Path, args: argparse.Namespace) -> dict[str, Any]:
    by_family: dict[str, Counter[str]] = defaultdict(Counter)
    examples: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for result in results:
        stats = by_family[result.family]
        stats["total"] += 1
        stats["clean"] += int(result.clean)
        stats["visible"] += int(result.visible)
        stats["notdef"] += int(result.has_notdef)
        stats["dotted_circle"] += int(result.has_dotted_circle)
        stats["not_clean"] += int(not result.clean)
        if not result.clean and len(examples[result.family]) < 8:
            examples[result.family].append(asdict(result))
    families = {}
    for family, stats in sorted(by_family.items()):
        total = stats["total"]
        families[family] = {
            "total": total,
            "clean": stats["clean"],
            "clean_rate": stats["clean"] / total if total else 0.0,
            "visible": stats["visible"],
            "notdef": stats["notdef"],
            "dotted_circle": stats["dotted_circle"],
            "not_clean": stats["not_clean"],
            "not_clean_examples": examples.get(family, []),
        }
    component_classes = _classes()
    return {
        "schema": SCHEMA,
        "font_path": str(font_path),
        "font_sha256": sha256_file(font_path),
        "font_size": args.font_size,
        "unicode_data_version": unicodedata.unidata_version,
        "dark_threshold": args.dark_threshold,
        "total_cases": len(results),
        "overall": {
            "clean": sum(1 for result in results if result.clean),
            "visible": sum(1 for result in results if result.visible),
            "notdef": sum(1 for result in results if result.has_notdef),
            "dotted_circle": sum(1 for result in results if result.has_dotted_circle),
        },
        "component_classes": {
            field: [
                {
                    "char": ch,
                    "codepoint": f"U+{ord(ch):04X}",
                    "name": unicodedata.name(ch),
                    "category": unicodedata.category(ch),
                    "combining_class": unicodedata.combining(ch),
                    "bidi": unicodedata.bidirectional(ch),
                }
                for ch in getattr(component_classes, field)
            ]
            for field in (
                "bases",
                "vowel_signs",
                "subjoined",
                "finals",
                "signs",
                "symbols",
                "punctuation",
                "digits",
            )
        },
        "families": families,
    }


def _write_results_csv(path: Path, results: list[ShapeResult]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    if not results:
        raise DataValidationError("no shaping results to write")
    with path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=list(asdict(results[0]).keys()))
        writer.writeheader()
        for result in results:
            writer.writerow(asdict(result))


def _write_summary_md(path: Path, summary: dict[str, Any]) -> None:
    lines = [
        "# Limbu Unicode Shaping Analysis",
        "",
        f"- font: `{summary['font_path']}`",
        f"- cases: `{summary['total_cases']}`",
        f"- clean: `{summary['overall']['clean']}`",
        f"- visible: `{summary['overall']['visible']}`",
        f"- dotted-circle cases: `{summary['overall']['dotted_circle']}`",
        f"- .notdef cases: `{summary['overall']['notdef']}`",
        "",
        "## Families",
        "",
        "| Family | Total | Clean | Clean rate | Dotted | Notdef |",
        "| --- | ---: | ---: | ---: | ---: | ---: |",
    ]
    for family, stats in summary["families"].items():
        lines.append(
            f"| `{family}` | {stats['total']} | {stats['clean']} | "
            f"{stats['clean_rate']:.4f} | {stats['dotted_circle']} | {stats['notdef']} |"
        )
    lines.extend(
        [
            "",
            "Clean means visible ink, no `.notdef`, and no dotted-circle glyph in HarfBuzz",
            "shaping with the tested font. Use these family-level results to constrain",
            "synthetic generation order before adding coverage rows.",
            "",
        ]
    )
    path.write_text("\n".join(lines), encoding="utf-8")


def analyze(args: argparse.Namespace) -> dict[str, Any]:
    font_path = Path(args.font)
    probe = LimbuShapeProbe(font_path, args.font_size, args.dark_threshold)
    cases = _build_cases(args)
    results = [probe.analyze(family, text) for family, text in cases]
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    _write_results_csv(out / "limbu-shaping-cases.csv", results)
    summary = _summarize(results, font_path, args)
    (out / "limbu-shaping-summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    _write_summary_md(out / "limbu-shaping-summary.md", summary)
    return summary


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--font", type=Path, default=DEFAULT_FONT)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--font-size", type=int, default=42)
    parser.add_argument("--dark-threshold", type=int, default=180)
    parser.add_argument("--random-order-samples", type=int, default=5000)
    parser.add_argument("--random-canonical-samples", type=int, default=1000)
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--seed", type=int, default=240624)
    return parser.parse_args(argv)


def main(argv: list[str] | None = None) -> int:
    summary = analyze(parse_args(argv))
    print(json.dumps(summary, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())