Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/analyze_limbu_unicode_shaping.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_limbu_unicode_shaping.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/analyze_limbu_unicode_shaping.py
-
curl -L -o analyze_limbu_unicode_shaping.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_limbu_unicode_shaping.py
13.8 kB
| #!/usr/bin/env python3 | |
| """Analyze Unicode Limbu component classes and diacritic shaping behavior.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import itertools | |
| import json | |
| import random | |
| import unicodedata | |
| from collections import Counter, defaultdict | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from typing import Any, Iterable | |
| try: | |
| import uharfbuzz as hb | |
| from fontTools.ttLib import TTFont | |
| from PIL import Image, ImageDraw, ImageFont | |
| except ImportError as exc: # pragma: no cover - environment guard | |
| raise SystemExit(f"uharfbuzz, fontTools, and Pillow are required: {exc}") from exc | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.manifest import sha256_file | |
| DEFAULT_FONT = Path("data/fonts/namdhinggo/Namdhinggo-3.100/Namdhinggo-Regular.ttf") | |
| SCHEMA = "ocrtech.limbu-unicode-shaping-analysis.v1" | |
| class ComponentClasses: | |
| bases: list[str] | |
| vowel_signs: list[str] | |
| subjoined: list[str] | |
| finals: list[str] | |
| signs: list[str] | |
| symbols: list[str] | |
| punctuation: list[str] | |
| digits: list[str] | |
| class ShapeResult: | |
| family: str | |
| text: str | |
| codepoints: str | |
| names: str | |
| clean: bool | |
| visible: bool | |
| glyph_count: int | |
| has_notdef: bool | |
| has_dotted_circle: bool | |
| dark_pixels: int | |
| min_gray: int | |
| bbox_width: int | |
| bbox_height: int | |
| def _assigned_limbu_chars() -> list[str]: | |
| chars: list[str] = [] | |
| for cp in range(0x1900, 0x1950): | |
| ch = chr(cp) | |
| try: | |
| unicodedata.name(ch) | |
| except ValueError: | |
| continue | |
| chars.append(ch) | |
| return chars | |
| def _classes() -> ComponentClasses: | |
| chars = _assigned_limbu_chars() | |
| bases = [ch for ch in chars if 0x1900 <= ord(ch) <= 0x191E] | |
| vowel_signs = [ch for ch in chars if 0x1920 <= ord(ch) <= 0x1928] | |
| subjoined = [ch for ch in chars if 0x1929 <= ord(ch) <= 0x192B] | |
| finals = [ch for ch in chars if 0x1930 <= ord(ch) <= 0x1938] | |
| signs = [ch for ch in chars if 0x1939 <= ord(ch) <= 0x193B] | |
| symbols = [ch for ch in chars if ord(ch) == 0x1940] | |
| punctuation = [ch for ch in chars if 0x1944 <= ord(ch) <= 0x1945] | |
| digits = [ch for ch in chars if 0x1946 <= ord(ch) <= 0x194F] | |
| return ComponentClasses(bases, vowel_signs, subjoined, finals, signs, symbols, punctuation, digits) | |
| class LimbuShapeProbe: | |
| def __init__(self, font_path: Path, font_size: int, dark_threshold: int): | |
| if not font_path.is_file(): | |
| raise DataValidationError(f"font does not exist: {font_path}") | |
| self.font_path = font_path | |
| self.font_size = font_size | |
| self.dark_threshold = dark_threshold | |
| self.font_data = font_path.read_bytes() | |
| self.hb_font = hb.Font(hb.Face(self.font_data)) | |
| self.pil_font = ImageFont.truetype(str(font_path), font_size, layout_engine=ImageFont.Layout.RAQM) | |
| tt = TTFont(str(font_path)) | |
| order = tt.getGlyphOrder() | |
| cmap = tt.getBestCmap() or {} | |
| self.bad_glyph_ids = {0} | |
| self.dotted_gid = None | |
| if 0x25CC in cmap: | |
| self.dotted_gid = order.index(cmap[0x25CC]) | |
| self.bad_glyph_ids.add(self.dotted_gid) | |
| def shape(self, text: str) -> tuple[int, bool, bool]: | |
| buf = hb.Buffer() | |
| buf.add_str(text) | |
| buf.guess_segment_properties() | |
| hb.shape(self.hb_font, buf, {}) | |
| glyph_ids = [info.codepoint for info in buf.glyph_infos] | |
| has_notdef = any(gid == 0 for gid in glyph_ids) | |
| has_dotted = self.dotted_gid is not None and any(gid == self.dotted_gid for gid in glyph_ids) | |
| return len(glyph_ids), has_notdef, has_dotted | |
| def render_stats(self, text: str) -> tuple[int, int, int, int]: | |
| probe = Image.new("RGB", (2048, 256), "white") | |
| draw = ImageDraw.Draw(probe) | |
| bbox = draw.textbbox((16, 32), text, font=self.pil_font) | |
| draw.text((16, 32), text, font=self.pil_font, fill=(0, 0, 0)) | |
| gray = probe.convert("L") | |
| histogram = gray.histogram() | |
| dark_pixels = sum(histogram[: self.dark_threshold]) | |
| min_gray = next((value for value, count in enumerate(histogram) if count), 255) | |
| width = max(0, bbox[2] - bbox[0]) | |
| height = max(0, bbox[3] - bbox[1]) | |
| return dark_pixels, min_gray, width, height | |
| def analyze(self, family: str, text: str) -> ShapeResult: | |
| glyph_count, has_notdef, has_dotted = self.shape(text) | |
| dark_pixels, min_gray, width, height = self.render_stats(text) | |
| visible = dark_pixels > 0 and height > 0 and width > 0 | |
| clean = visible and not has_notdef and not has_dotted | |
| return ShapeResult( | |
| family=family, | |
| text=text, | |
| codepoints=" ".join(f"U+{ord(ch):04X}" for ch in text), | |
| names="; ".join(unicodedata.name(ch, "<no name>") for ch in text), | |
| clean=clean, | |
| visible=visible, | |
| glyph_count=glyph_count, | |
| has_notdef=has_notdef, | |
| has_dotted_circle=has_dotted, | |
| dark_pixels=dark_pixels, | |
| min_gray=min_gray, | |
| bbox_width=width, | |
| bbox_height=height, | |
| ) | |
| def _add_family(cases: list[tuple[str, str]], family: str, texts: Iterable[str]) -> None: | |
| for text in texts: | |
| cases.append((family, text)) | |
| def _build_cases(args: argparse.Namespace) -> list[tuple[str, str]]: | |
| c = _classes() | |
| cases: list[tuple[str, str]] = [] | |
| _add_family(cases, "single_assigned", _assigned_limbu_chars()) | |
| _add_family(cases, "base_vowel", (b + v for b in c.bases for v in c.vowel_signs)) | |
| _add_family(cases, "base_subjoined", (b + s for b in c.bases for s in c.subjoined)) | |
| _add_family(cases, "base_final", (b + f for b in c.bases for f in c.finals)) | |
| _add_family(cases, "base_sign", (b + s for b in c.bases for s in c.signs)) | |
| _add_family(cases, "base_vowel_final", (b + v + f for b in c.bases for v in c.vowel_signs for f in c.finals)) | |
| _add_family(cases, "base_vowel_sign", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.signs)) | |
| _add_family(cases, "base_final_sign", (b + f + s for b in c.bases for f in c.finals for s in c.signs)) | |
| _add_family( | |
| cases, | |
| "base_vowel_final_sign", | |
| (b + v + f + s for b in c.bases for v in c.vowel_signs for f in c.finals for s in c.signs), | |
| ) | |
| _add_family(cases, "base_subjoined_vowel", (b + s + v for b in c.bases for s in c.subjoined for v in c.vowel_signs)) | |
| _add_family(cases, "base_vowel_subjoined", (b + v + s for b in c.bases for v in c.vowel_signs for s in c.subjoined)) | |
| _add_family(cases, "base_subjoined_final", (b + s + f for b in c.bases for s in c.subjoined for f in c.finals)) | |
| _add_family(cases, "base_final_subjoined", (b + f + s for b in c.bases for f in c.finals for s in c.subjoined)) | |
| _add_family( | |
| cases, | |
| "punct_symbol_digit_context", | |
| ("ᤁᤡ " + ch + " ᤕᤠ" for ch in c.symbols + c.punctuation + c.digits), | |
| ) | |
| rng = random.Random(args.seed) | |
| components = c.vowel_signs + c.subjoined + c.finals + c.signs | |
| for index in range(args.random_order_samples): | |
| b = rng.choice(c.bases) | |
| chosen = [rng.choice(c.vowel_signs), rng.choice(c.subjoined), rng.choice(c.finals), rng.choice(c.signs)] | |
| rng.shuffle(chosen) | |
| cases.append(("random_component_order", b + "".join(chosen))) | |
| if index < args.random_canonical_samples: | |
| cases.append( | |
| ( | |
| "random_canonical_order", | |
| b + rng.choice(c.vowel_signs) + rng.choice(c.subjoined) + rng.choice(c.finals) + rng.choice(c.signs), | |
| ) | |
| ) | |
| if args.limit and len(cases) > args.limit: | |
| rng.shuffle(cases) | |
| cases = cases[: args.limit] | |
| return cases | |
| def _summarize(results: list[ShapeResult], font_path: Path, args: argparse.Namespace) -> dict[str, Any]: | |
| by_family: dict[str, Counter[str]] = defaultdict(Counter) | |
| examples: dict[str, list[dict[str, Any]]] = defaultdict(list) | |
| for result in results: | |
| stats = by_family[result.family] | |
| stats["total"] += 1 | |
| stats["clean"] += int(result.clean) | |
| stats["visible"] += int(result.visible) | |
| stats["notdef"] += int(result.has_notdef) | |
| stats["dotted_circle"] += int(result.has_dotted_circle) | |
| stats["not_clean"] += int(not result.clean) | |
| if not result.clean and len(examples[result.family]) < 8: | |
| examples[result.family].append(asdict(result)) | |
| families = {} | |
| for family, stats in sorted(by_family.items()): | |
| total = stats["total"] | |
| families[family] = { | |
| "total": total, | |
| "clean": stats["clean"], | |
| "clean_rate": stats["clean"] / total if total else 0.0, | |
| "visible": stats["visible"], | |
| "notdef": stats["notdef"], | |
| "dotted_circle": stats["dotted_circle"], | |
| "not_clean": stats["not_clean"], | |
| "not_clean_examples": examples.get(family, []), | |
| } | |
| component_classes = _classes() | |
| return { | |
| "schema": SCHEMA, | |
| "font_path": str(font_path), | |
| "font_sha256": sha256_file(font_path), | |
| "font_size": args.font_size, | |
| "unicode_data_version": unicodedata.unidata_version, | |
| "dark_threshold": args.dark_threshold, | |
| "total_cases": len(results), | |
| "overall": { | |
| "clean": sum(1 for result in results if result.clean), | |
| "visible": sum(1 for result in results if result.visible), | |
| "notdef": sum(1 for result in results if result.has_notdef), | |
| "dotted_circle": sum(1 for result in results if result.has_dotted_circle), | |
| }, | |
| "component_classes": { | |
| field: [ | |
| { | |
| "char": ch, | |
| "codepoint": f"U+{ord(ch):04X}", | |
| "name": unicodedata.name(ch), | |
| "category": unicodedata.category(ch), | |
| "combining_class": unicodedata.combining(ch), | |
| "bidi": unicodedata.bidirectional(ch), | |
| } | |
| for ch in getattr(component_classes, field) | |
| ] | |
| for field in ( | |
| "bases", | |
| "vowel_signs", | |
| "subjoined", | |
| "finals", | |
| "signs", | |
| "symbols", | |
| "punctuation", | |
| "digits", | |
| ) | |
| }, | |
| "families": families, | |
| } | |
| def _write_results_csv(path: Path, results: list[ShapeResult]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| if not results: | |
| raise DataValidationError("no shaping results to write") | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=list(asdict(results[0]).keys())) | |
| writer.writeheader() | |
| for result in results: | |
| writer.writerow(asdict(result)) | |
| def _write_summary_md(path: Path, summary: dict[str, Any]) -> None: | |
| lines = [ | |
| "# Limbu Unicode Shaping Analysis", | |
| "", | |
| f"- font: `{summary['font_path']}`", | |
| f"- cases: `{summary['total_cases']}`", | |
| f"- clean: `{summary['overall']['clean']}`", | |
| f"- visible: `{summary['overall']['visible']}`", | |
| f"- dotted-circle cases: `{summary['overall']['dotted_circle']}`", | |
| f"- .notdef cases: `{summary['overall']['notdef']}`", | |
| "", | |
| "## Families", | |
| "", | |
| "| Family | Total | Clean | Clean rate | Dotted | Notdef |", | |
| "| --- | ---: | ---: | ---: | ---: | ---: |", | |
| ] | |
| for family, stats in summary["families"].items(): | |
| lines.append( | |
| f"| `{family}` | {stats['total']} | {stats['clean']} | " | |
| f"{stats['clean_rate']:.4f} | {stats['dotted_circle']} | {stats['notdef']} |" | |
| ) | |
| lines.extend( | |
| [ | |
| "", | |
| "Clean means visible ink, no `.notdef`, and no dotted-circle glyph in HarfBuzz", | |
| "shaping with the tested font. Use these family-level results to constrain", | |
| "synthetic generation order before adding coverage rows.", | |
| "", | |
| ] | |
| ) | |
| path.write_text("\n".join(lines), encoding="utf-8") | |
| def analyze(args: argparse.Namespace) -> dict[str, Any]: | |
| font_path = Path(args.font) | |
| probe = LimbuShapeProbe(font_path, args.font_size, args.dark_threshold) | |
| cases = _build_cases(args) | |
| results = [probe.analyze(family, text) for family, text in cases] | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| _write_results_csv(out / "limbu-shaping-cases.csv", results) | |
| summary = _summarize(results, font_path, args) | |
| (out / "limbu-shaping-summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_summary_md(out / "limbu-shaping-summary.md", summary) | |
| return summary | |
| def parse_args(argv: list[str] | None = None) -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--font", type=Path, default=DEFAULT_FONT) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--font-size", type=int, default=42) | |
| parser.add_argument("--dark-threshold", type=int, default=180) | |
| parser.add_argument("--random-order-samples", type=int, default=5000) | |
| parser.add_argument("--random-canonical-samples", type=int, default=1000) | |
| parser.add_argument("--limit", type=int, default=0) | |
| parser.add_argument("--seed", type=int, default=240624) | |
| return parser.parse_args(argv) | |
| def main(argv: list[str] | None = None) -> int: | |
| summary = analyze(parse_args(argv)) | |
| print(json.dumps(summary, ensure_ascii=False, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |