#!/usr/bin/env python3 """Analyze Unicode Limbu component classes and diacritic shaping behavior.""" from __future__ import annotations import argparse import csv import itertools import json import random import unicodedata from collections import Counter, defaultdict from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Iterable try: import uharfbuzz as hb from fontTools.ttLib import TTFont from PIL import Image, ImageDraw, ImageFont except ImportError as exc: # pragma: no cover - environment guard raise SystemExit(f"uharfbuzz, fontTools, and Pillow are required: {exc}") from exc from ocrtech.errors import DataValidationError from ocrtech.manifest import sha256_file DEFAULT_FONT = Path("data/fonts/namdhinggo/Namdhinggo-3.100/Namdhinggo-Regular.ttf") SCHEMA = "ocrtech.limbu-unicode-shaping-analysis.v1" @dataclass(slots=True) class ComponentClasses: bases: list[str] vowel_signs: list[str] subjoined: list[str] finals: list[str] signs: list[str] symbols: list[str] punctuation: list[str] digits: list[str] @dataclass(slots=True) class ShapeResult: family: str text: str codepoints: str names: str clean: bool visible: bool glyph_count: int has_notdef: bool has_dotted_circle: bool dark_pixels: int min_gray: int bbox_width: int bbox_height: int def _assigned_limbu_chars() -> list[str]: chars: list[str] = [] for cp in range(0x1900, 0x1950): ch = chr(cp) try: unicodedata.name(ch) except ValueError: continue chars.append(ch) return chars def _classes() -> ComponentClasses: chars = _assigned_limbu_chars() bases = [ch for ch in chars if 0x1900 <= ord(ch) <= 0x191E] vowel_signs = [ch for ch in chars if 0x1920 <= ord(ch) <= 0x1928] subjoined = [ch for ch in chars if 0x1929 <= ord(ch) <= 0x192B] finals = [ch for ch in chars if 0x1930 <= ord(ch) <= 0x1938] signs = [ch for ch in chars if 0x1939 <= ord(ch) <= 0x193B] symbols = [ch for ch in chars if ord(ch) == 0x1940] punctuation = [ch for ch in chars if 0x1944 <= ord(ch) <= 0x1945] digits = [ch for ch in chars if 0x1946 <= ord(ch) <= 0x194F] return ComponentClasses(bases, vowel_signs, subjoined, finals, signs, symbols, punctuation, digits) class LimbuShapeProbe: def __init__(self, font_path: Path, font_size: int, dark_threshold: int): if not font_path.is_file(): raise DataValidationError(f"font does not exist: {font_path}") self.font_path = font_path self.font_size = font_size self.dark_threshold = dark_threshold self.font_data = font_path.read_bytes() self.hb_font = hb.Font(hb.Face(self.font_data)) self.pil_font = ImageFont.truetype(str(font_path), font_size, layout_engine=ImageFont.Layout.RAQM) tt = TTFont(str(font_path)) order = tt.getGlyphOrder() cmap = tt.getBestCmap() or {} self.bad_glyph_ids = {0} self.dotted_gid = None if 0x25CC in cmap: self.dotted_gid = order.index(cmap[0x25CC]) self.bad_glyph_ids.add(self.dotted_gid) def shape(self, text: str) -> tuple[int, bool, bool]: buf = hb.Buffer() buf.add_str(text) buf.guess_segment_properties() hb.shape(self.hb_font, buf, {}) glyph_ids = [info.codepoint for info in buf.glyph_infos] has_notdef = any(gid == 0 for gid in glyph_ids) has_dotted = self.dotted_gid is not None and any(gid == self.dotted_gid for gid in glyph_ids) return len(glyph_ids), has_notdef, has_dotted def render_stats(self, text: str) -> tuple[int, int, int, int]: probe = Image.new("RGB", (2048, 256), "white") draw = ImageDraw.Draw(probe) bbox = draw.textbbox((16, 32), text, font=self.pil_font) draw.text((16, 32), text, font=self.pil_font, fill=(0, 0, 0)) gray = probe.convert("L") histogram = gray.histogram() dark_pixels = sum(histogram[: self.dark_threshold]) min_gray = next((value for value, count in enumerate(histogram) if count), 255) width = max(0, bbox[2] - bbox[0]) height = max(0, bbox[3] - bbox[1]) return dark_pixels, min_gray, width, height def analyze(self, family: str, text: str) -> ShapeResult: glyph_count, has_notdef, has_dotted = self.shape(text) dark_pixels, min_gray, width, height = self.render_stats(text) visible = dark_pixels > 0 and height > 0 and width > 0 clean = visible and not has_notdef and not has_dotted return ShapeResult( family=family, text=text, codepoints=" ".join(f"U+{ord(ch):04X}" for ch in text), names="; ".join(unicodedata.name(ch, "") for ch in text), clean=clean, visible=visible, glyph_count=glyph_count, has_notdef=has_notdef, has_dotted_circle=has_dotted, dark_pixels=dark_pixels, min_gray=min_gray, bbox_width=width, bbox_height=height, ) def _add_family(cases: list[tuple[str, str]], family: str, texts: Iterable[str]) -> None: for text in texts: cases.append((family, text)) def _build_cases(args: argparse.Namespace) -> list[tuple[str, str]]: c = _classes() cases: list[tuple[str, str]] = [] _add_family(cases, "single_assigned", _assigned_limbu_chars()) _add_family(cases, "base_vowel", (b + v for b in c.bases for v in c.vowel_signs)) _add_family(cases, "base_subjoined", (b + s for b in c.bases for s in c.subjoined)) _add_family(cases, "base_final", (b + f for b in c.bases for f in c.finals)) _add_family(cases, "base_sign", (b + s for b in c.bases for s in c.signs)) _add_family(cases, "base_vowel_final", (b + v + f for b in c.bases for v in c.vowel_signs for f in c.finals)) _add_family(cases, "base_vowel_sign", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.signs)) _add_family(cases, "base_final_sign", (b + f + s for b in c.bases for f in c.finals for s in c.signs)) _add_family( cases, "base_vowel_final_sign", (b + v + f + s for b in c.bases for v in c.vowel_signs for f in c.finals for s in c.signs), ) _add_family(cases, "base_subjoined_vowel", (b + s + v for b in c.bases for s in c.subjoined for v in c.vowel_signs)) _add_family(cases, "base_vowel_subjoined", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.subjoined)) _add_family(cases, "base_subjoined_final", (b + s + f for b in c.bases for s in c.subjoined for f in c.finals)) _add_family(cases, "base_final_subjoined", (b + f + s for b in c.bases for f in c.finals for s in c.subjoined)) _add_family( cases, "punct_symbol_digit_context", ("ᤁᤡ " + ch + " ᤕᤠ" for ch in c.symbols + c.punctuation + c.digits), ) rng = random.Random(args.seed) components = c.vowel_signs + c.subjoined + c.finals + c.signs for index in range(args.random_order_samples): b = rng.choice(c.bases) chosen = [rng.choice(c.vowel_signs), rng.choice(c.subjoined), rng.choice(c.finals), rng.choice(c.signs)] rng.shuffle(chosen) cases.append(("random_component_order", b + "".join(chosen))) if index < args.random_canonical_samples: cases.append( ( "random_canonical_order", b + rng.choice(c.vowel_signs) + rng.choice(c.subjoined) + rng.choice(c.finals) + rng.choice(c.signs), ) ) if args.limit and len(cases) > args.limit: rng.shuffle(cases) cases = cases[: args.limit] return cases def _summarize(results: list[ShapeResult], font_path: Path, args: argparse.Namespace) -> dict[str, Any]: by_family: dict[str, Counter[str]] = defaultdict(Counter) examples: dict[str, list[dict[str, Any]]] = defaultdict(list) for result in results: stats = by_family[result.family] stats["total"] += 1 stats["clean"] += int(result.clean) stats["visible"] += int(result.visible) stats["notdef"] += int(result.has_notdef) stats["dotted_circle"] += int(result.has_dotted_circle) stats["not_clean"] += int(not result.clean) if not result.clean and len(examples[result.family]) < 8: examples[result.family].append(asdict(result)) families = {} for family, stats in sorted(by_family.items()): total = stats["total"] families[family] = { "total": total, "clean": stats["clean"], "clean_rate": stats["clean"] / total if total else 0.0, "visible": stats["visible"], "notdef": stats["notdef"], "dotted_circle": stats["dotted_circle"], "not_clean": stats["not_clean"], "not_clean_examples": examples.get(family, []), } component_classes = _classes() return { "schema": SCHEMA, "font_path": str(font_path), "font_sha256": sha256_file(font_path), "font_size": args.font_size, "unicode_data_version": unicodedata.unidata_version, "dark_threshold": args.dark_threshold, "total_cases": len(results), "overall": { "clean": sum(1 for result in results if result.clean), "visible": sum(1 for result in results if result.visible), "notdef": sum(1 for result in results if result.has_notdef), "dotted_circle": sum(1 for result in results if result.has_dotted_circle), }, "component_classes": { field: [ { "char": ch, "codepoint": f"U+{ord(ch):04X}", "name": unicodedata.name(ch), "category": unicodedata.category(ch), "combining_class": unicodedata.combining(ch), "bidi": unicodedata.bidirectional(ch), } for ch in getattr(component_classes, field) ] for field in ( "bases", "vowel_signs", "subjoined", "finals", "signs", "symbols", "punctuation", "digits", ) }, "families": families, } def _write_results_csv(path: Path, results: list[ShapeResult]) -> None: path.parent.mkdir(parents=True, exist_ok=True) if not results: raise DataValidationError("no shaping results to write") with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=list(asdict(results[0]).keys())) writer.writeheader() for result in results: writer.writerow(asdict(result)) def _write_summary_md(path: Path, summary: dict[str, Any]) -> None: lines = [ "# Limbu Unicode Shaping Analysis", "", f"- font: `{summary['font_path']}`", f"- cases: `{summary['total_cases']}`", f"- clean: `{summary['overall']['clean']}`", f"- visible: `{summary['overall']['visible']}`", f"- dotted-circle cases: `{summary['overall']['dotted_circle']}`", f"- .notdef cases: `{summary['overall']['notdef']}`", "", "## Families", "", "| Family | Total | Clean | Clean rate | Dotted | Notdef |", "| --- | ---: | ---: | ---: | ---: | ---: |", ] for family, stats in summary["families"].items(): lines.append( f"| `{family}` | {stats['total']} | {stats['clean']} | " f"{stats['clean_rate']:.4f} | {stats['dotted_circle']} | {stats['notdef']} |" ) lines.extend( [ "", "Clean means visible ink, no `.notdef`, and no dotted-circle glyph in HarfBuzz", "shaping with the tested font. Use these family-level results to constrain", "synthetic generation order before adding coverage rows.", "", ] ) path.write_text("\n".join(lines), encoding="utf-8") def analyze(args: argparse.Namespace) -> dict[str, Any]: font_path = Path(args.font) probe = LimbuShapeProbe(font_path, args.font_size, args.dark_threshold) cases = _build_cases(args) results = [probe.analyze(family, text) for family, text in cases] out = Path(args.out) out.mkdir(parents=True, exist_ok=True) _write_results_csv(out / "limbu-shaping-cases.csv", results) summary = _summarize(results, font_path, args) (out / "limbu-shaping-summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") _write_summary_md(out / "limbu-shaping-summary.md", summary) return summary def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--font", type=Path, default=DEFAULT_FONT) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--font-size", type=int, default=42) parser.add_argument("--dark-threshold", type=int, default=180) parser.add_argument("--random-order-samples", type=int, default=5000) parser.add_argument("--random-canonical-samples", type=int, default=1000) parser.add_argument("--limit", type=int, default=0) parser.add_argument("--seed", type=int, default=240624) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: summary = analyze(parse_args(argv)) print(json.dumps(summary, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())