ocr-projects-archive / ocr-tech /tools /analyze_limbu_unicode_shaping.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
13.8 kB
#!/usr/bin/env python3
"""Analyze Unicode Limbu component classes and diacritic shaping behavior."""
from __future__ import annotations
import argparse
import csv
import itertools
import json
import random
import unicodedata
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Iterable
try:
import uharfbuzz as hb
from fontTools.ttLib import TTFont
from PIL import Image, ImageDraw, ImageFont
except ImportError as exc: # pragma: no cover - environment guard
raise SystemExit(f"uharfbuzz, fontTools, and Pillow are required: {exc}") from exc
from ocrtech.errors import DataValidationError
from ocrtech.manifest import sha256_file
DEFAULT_FONT = Path("data/fonts/namdhinggo/Namdhinggo-3.100/Namdhinggo-Regular.ttf")
SCHEMA = "ocrtech.limbu-unicode-shaping-analysis.v1"
@dataclass(slots=True)
class ComponentClasses:
bases: list[str]
vowel_signs: list[str]
subjoined: list[str]
finals: list[str]
signs: list[str]
symbols: list[str]
punctuation: list[str]
digits: list[str]
@dataclass(slots=True)
class ShapeResult:
family: str
text: str
codepoints: str
names: str
clean: bool
visible: bool
glyph_count: int
has_notdef: bool
has_dotted_circle: bool
dark_pixels: int
min_gray: int
bbox_width: int
bbox_height: int
def _assigned_limbu_chars() -> list[str]:
chars: list[str] = []
for cp in range(0x1900, 0x1950):
ch = chr(cp)
try:
unicodedata.name(ch)
except ValueError:
continue
chars.append(ch)
return chars
def _classes() -> ComponentClasses:
chars = _assigned_limbu_chars()
bases = [ch for ch in chars if 0x1900 <= ord(ch) <= 0x191E]
vowel_signs = [ch for ch in chars if 0x1920 <= ord(ch) <= 0x1928]
subjoined = [ch for ch in chars if 0x1929 <= ord(ch) <= 0x192B]
finals = [ch for ch in chars if 0x1930 <= ord(ch) <= 0x1938]
signs = [ch for ch in chars if 0x1939 <= ord(ch) <= 0x193B]
symbols = [ch for ch in chars if ord(ch) == 0x1940]
punctuation = [ch for ch in chars if 0x1944 <= ord(ch) <= 0x1945]
digits = [ch for ch in chars if 0x1946 <= ord(ch) <= 0x194F]
return ComponentClasses(bases, vowel_signs, subjoined, finals, signs, symbols, punctuation, digits)
class LimbuShapeProbe:
def __init__(self, font_path: Path, font_size: int, dark_threshold: int):
if not font_path.is_file():
raise DataValidationError(f"font does not exist: {font_path}")
self.font_path = font_path
self.font_size = font_size
self.dark_threshold = dark_threshold
self.font_data = font_path.read_bytes()
self.hb_font = hb.Font(hb.Face(self.font_data))
self.pil_font = ImageFont.truetype(str(font_path), font_size, layout_engine=ImageFont.Layout.RAQM)
tt = TTFont(str(font_path))
order = tt.getGlyphOrder()
cmap = tt.getBestCmap() or {}
self.bad_glyph_ids = {0}
self.dotted_gid = None
if 0x25CC in cmap:
self.dotted_gid = order.index(cmap[0x25CC])
self.bad_glyph_ids.add(self.dotted_gid)
def shape(self, text: str) -> tuple[int, bool, bool]:
buf = hb.Buffer()
buf.add_str(text)
buf.guess_segment_properties()
hb.shape(self.hb_font, buf, {})
glyph_ids = [info.codepoint for info in buf.glyph_infos]
has_notdef = any(gid == 0 for gid in glyph_ids)
has_dotted = self.dotted_gid is not None and any(gid == self.dotted_gid for gid in glyph_ids)
return len(glyph_ids), has_notdef, has_dotted
def render_stats(self, text: str) -> tuple[int, int, int, int]:
probe = Image.new("RGB", (2048, 256), "white")
draw = ImageDraw.Draw(probe)
bbox = draw.textbbox((16, 32), text, font=self.pil_font)
draw.text((16, 32), text, font=self.pil_font, fill=(0, 0, 0))
gray = probe.convert("L")
histogram = gray.histogram()
dark_pixels = sum(histogram[: self.dark_threshold])
min_gray = next((value for value, count in enumerate(histogram) if count), 255)
width = max(0, bbox[2] - bbox[0])
height = max(0, bbox[3] - bbox[1])
return dark_pixels, min_gray, width, height
def analyze(self, family: str, text: str) -> ShapeResult:
glyph_count, has_notdef, has_dotted = self.shape(text)
dark_pixels, min_gray, width, height = self.render_stats(text)
visible = dark_pixels > 0 and height > 0 and width > 0
clean = visible and not has_notdef and not has_dotted
return ShapeResult(
family=family,
text=text,
codepoints=" ".join(f"U+{ord(ch):04X}" for ch in text),
names="; ".join(unicodedata.name(ch, "<no name>") for ch in text),
clean=clean,
visible=visible,
glyph_count=glyph_count,
has_notdef=has_notdef,
has_dotted_circle=has_dotted,
dark_pixels=dark_pixels,
min_gray=min_gray,
bbox_width=width,
bbox_height=height,
)
def _add_family(cases: list[tuple[str, str]], family: str, texts: Iterable[str]) -> None:
for text in texts:
cases.append((family, text))
def _build_cases(args: argparse.Namespace) -> list[tuple[str, str]]:
c = _classes()
cases: list[tuple[str, str]] = []
_add_family(cases, "single_assigned", _assigned_limbu_chars())
_add_family(cases, "base_vowel", (b + v for b in c.bases for v in c.vowel_signs))
_add_family(cases, "base_subjoined", (b + s for b in c.bases for s in c.subjoined))
_add_family(cases, "base_final", (b + f for b in c.bases for f in c.finals))
_add_family(cases, "base_sign", (b + s for b in c.bases for s in c.signs))
_add_family(cases, "base_vowel_final", (b + v + f for b in c.bases for v in c.vowel_signs for f in c.finals))
_add_family(cases, "base_vowel_sign", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.signs))
_add_family(cases, "base_final_sign", (b + f + s for b in c.bases for f in c.finals for s in c.signs))
_add_family(
cases,
"base_vowel_final_sign",
(b + v + f + s for b in c.bases for v in c.vowel_signs for f in c.finals for s in c.signs),
)
_add_family(cases, "base_subjoined_vowel", (b + s + v for b in c.bases for s in c.subjoined for v in c.vowel_signs))
_add_family(cases, "base_vowel_subjoined", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.subjoined))
_add_family(cases, "base_subjoined_final", (b + s + f for b in c.bases for s in c.subjoined for f in c.finals))
_add_family(cases, "base_final_subjoined", (b + f + s for b in c.bases for f in c.finals for s in c.subjoined))
_add_family(
cases,
"punct_symbol_digit_context",
("ᤁᤡ " + ch + " ᤕᤠ" for ch in c.symbols + c.punctuation + c.digits),
)
rng = random.Random(args.seed)
components = c.vowel_signs + c.subjoined + c.finals + c.signs
for index in range(args.random_order_samples):
b = rng.choice(c.bases)
chosen = [rng.choice(c.vowel_signs), rng.choice(c.subjoined), rng.choice(c.finals), rng.choice(c.signs)]
rng.shuffle(chosen)
cases.append(("random_component_order", b + "".join(chosen)))
if index < args.random_canonical_samples:
cases.append(
(
"random_canonical_order",
b + rng.choice(c.vowel_signs) + rng.choice(c.subjoined) + rng.choice(c.finals) + rng.choice(c.signs),
)
)
if args.limit and len(cases) > args.limit:
rng.shuffle(cases)
cases = cases[: args.limit]
return cases
def _summarize(results: list[ShapeResult], font_path: Path, args: argparse.Namespace) -> dict[str, Any]:
by_family: dict[str, Counter[str]] = defaultdict(Counter)
examples: dict[str, list[dict[str, Any]]] = defaultdict(list)
for result in results:
stats = by_family[result.family]
stats["total"] += 1
stats["clean"] += int(result.clean)
stats["visible"] += int(result.visible)
stats["notdef"] += int(result.has_notdef)
stats["dotted_circle"] += int(result.has_dotted_circle)
stats["not_clean"] += int(not result.clean)
if not result.clean and len(examples[result.family]) < 8:
examples[result.family].append(asdict(result))
families = {}
for family, stats in sorted(by_family.items()):
total = stats["total"]
families[family] = {
"total": total,
"clean": stats["clean"],
"clean_rate": stats["clean"] / total if total else 0.0,
"visible": stats["visible"],
"notdef": stats["notdef"],
"dotted_circle": stats["dotted_circle"],
"not_clean": stats["not_clean"],
"not_clean_examples": examples.get(family, []),
}
component_classes = _classes()
return {
"schema": SCHEMA,
"font_path": str(font_path),
"font_sha256": sha256_file(font_path),
"font_size": args.font_size,
"unicode_data_version": unicodedata.unidata_version,
"dark_threshold": args.dark_threshold,
"total_cases": len(results),
"overall": {
"clean": sum(1 for result in results if result.clean),
"visible": sum(1 for result in results if result.visible),
"notdef": sum(1 for result in results if result.has_notdef),
"dotted_circle": sum(1 for result in results if result.has_dotted_circle),
},
"component_classes": {
field: [
{
"char": ch,
"codepoint": f"U+{ord(ch):04X}",
"name": unicodedata.name(ch),
"category": unicodedata.category(ch),
"combining_class": unicodedata.combining(ch),
"bidi": unicodedata.bidirectional(ch),
}
for ch in getattr(component_classes, field)
]
for field in (
"bases",
"vowel_signs",
"subjoined",
"finals",
"signs",
"symbols",
"punctuation",
"digits",
)
},
"families": families,
}
def _write_results_csv(path: Path, results: list[ShapeResult]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if not results:
raise DataValidationError("no shaping results to write")
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(asdict(results[0]).keys()))
writer.writeheader()
for result in results:
writer.writerow(asdict(result))
def _write_summary_md(path: Path, summary: dict[str, Any]) -> None:
lines = [
"# Limbu Unicode Shaping Analysis",
"",
f"- font: `{summary['font_path']}`",
f"- cases: `{summary['total_cases']}`",
f"- clean: `{summary['overall']['clean']}`",
f"- visible: `{summary['overall']['visible']}`",
f"- dotted-circle cases: `{summary['overall']['dotted_circle']}`",
f"- .notdef cases: `{summary['overall']['notdef']}`",
"",
"## Families",
"",
"| Family | Total | Clean | Clean rate | Dotted | Notdef |",
"| --- | ---: | ---: | ---: | ---: | ---: |",
]
for family, stats in summary["families"].items():
lines.append(
f"| `{family}` | {stats['total']} | {stats['clean']} | "
f"{stats['clean_rate']:.4f} | {stats['dotted_circle']} | {stats['notdef']} |"
)
lines.extend(
[
"",
"Clean means visible ink, no `.notdef`, and no dotted-circle glyph in HarfBuzz",
"shaping with the tested font. Use these family-level results to constrain",
"synthetic generation order before adding coverage rows.",
"",
]
)
path.write_text("\n".join(lines), encoding="utf-8")
def analyze(args: argparse.Namespace) -> dict[str, Any]:
font_path = Path(args.font)
probe = LimbuShapeProbe(font_path, args.font_size, args.dark_threshold)
cases = _build_cases(args)
results = [probe.analyze(family, text) for family, text in cases]
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
_write_results_csv(out / "limbu-shaping-cases.csv", results)
summary = _summarize(results, font_path, args)
(out / "limbu-shaping-summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_summary_md(out / "limbu-shaping-summary.md", summary)
return summary
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--font", type=Path, default=DEFAULT_FONT)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--font-size", type=int, default=42)
parser.add_argument("--dark-threshold", type=int, default=180)
parser.add_argument("--random-order-samples", type=int, default=5000)
parser.add_argument("--random-canonical-samples", type=int, default=1000)
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--seed", type=int, default=240624)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
summary = analyze(parse_args(argv))
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())