Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/analyze_limbu_unicode_corpus_stats.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_limbu_unicode_corpus_stats.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/analyze_limbu_unicode_corpus_stats.py
-
curl -L -o analyze_limbu_unicode_corpus_stats.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_limbu_unicode_corpus_stats.py
14.4 kB
| #!/usr/bin/env python3 | |
| """Audit Limbu Unicode coverage and diacritic order in OCR label corpora.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import unicodedata | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| from typing import Any, Iterable | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.manifest import read_jsonl, sha256_file | |
| SCHEMA = "ocrtech.limbu-unicode-corpus-stats.v1" | |
| LIMBU_RANGE = range(0x1900, 0x1950) | |
| CLASS_ORDER = ("base", "vowel", "subjoined", "final", "sign") | |
| UNDERREPRESENTED_THRESHOLDS = (1, 10, 100, 250, 500, 1000) | |
| SIGN_MUKPHRENG = "\u1939" | |
| def _assigned_limbu_chars() -> list[str]: | |
| chars: list[str] = [] | |
| for cp in LIMBU_RANGE: | |
| ch = chr(cp) | |
| try: | |
| unicodedata.name(ch) | |
| except ValueError: | |
| continue | |
| chars.append(ch) | |
| return chars | |
| def _limbu_class(ch: str) -> str | None: | |
| cp = ord(ch) | |
| if 0x1900 <= cp <= 0x191E: | |
| return "base" | |
| if 0x1920 <= cp <= 0x1928: | |
| return "vowel" | |
| if 0x1929 <= cp <= 0x192B: | |
| return "subjoined" | |
| if 0x1930 <= cp <= 0x1938: | |
| return "final" | |
| if 0x1939 <= cp <= 0x193B: | |
| return "sign" | |
| if cp == 0x1940: | |
| return "symbol" | |
| if 0x1944 <= cp <= 0x1945: | |
| return "punctuation" | |
| if 0x1946 <= cp <= 0x194F: | |
| return "digit" | |
| return None | |
| def _char_record(ch: str, count: int, rows: int) -> dict[str, Any]: | |
| return { | |
| "char": ch, | |
| "codepoint": f"U+{ord(ch):04X}", | |
| "name": unicodedata.name(ch), | |
| "class": _limbu_class(ch), | |
| "category": unicodedata.category(ch), | |
| "combining_class": unicodedata.combining(ch), | |
| "bidi": unicodedata.bidirectional(ch), | |
| "count": count, | |
| "rows": rows, | |
| } | |
| def _signature(classes: Iterable[str]) -> str: | |
| return "+".join(classes) | |
| def _cluster_record(text: str, classes: list[str]) -> dict[str, Any]: | |
| class_counts = Counter(classes) | |
| has_mukphreng_after_final = any( | |
| left_class == "final" and right_char == SIGN_MUKPHRENG | |
| for left_class, right_char in zip(classes, text[1:]) | |
| ) | |
| return { | |
| "text": text, | |
| "codepoints": " ".join(f"U+{ord(ch):04X}" for ch in text), | |
| "names": "; ".join(unicodedata.name(ch, "<no name>") for ch in text), | |
| "signature": _signature(classes), | |
| "class_counts": dict(class_counts), | |
| "has_duplicate_vowel": class_counts["vowel"] > 1, | |
| "has_duplicate_subjoined": class_counts["subjoined"] > 1, | |
| "has_duplicate_final": class_counts["final"] > 1, | |
| "has_duplicate_sign": class_counts["sign"] > 1, | |
| "has_mukphreng_after_final": has_mukphreng_after_final, | |
| "normalization_changed_nfc": unicodedata.normalize("NFC", text) != text, | |
| "normalization_changed_nfd": unicodedata.normalize("NFD", text) != text, | |
| } | |
| def _add_example(bucket: dict[str, list[dict[str, Any]]], key: str, value: dict[str, Any], limit: int) -> None: | |
| if len(bucket[key]) < limit: | |
| bucket[key].append(value) | |
| class CorpusStats: | |
| def __init__(self, example_limit: int): | |
| self.example_limit = example_limit | |
| self.rows = 0 | |
| self.nfc_changed_rows = 0 | |
| self.nfd_changed_rows = 0 | |
| self.total_chars = 0 | |
| self.limbu_chars = 0 | |
| self.char_counts: Counter[str] = Counter() | |
| self.row_counts: Counter[str] = Counter() | |
| self.class_counts: Counter[str] = Counter() | |
| self.cluster_signatures: Counter[str] = Counter() | |
| self.cluster_class_sets: Counter[str] = Counter() | |
| self.cluster_count = 0 | |
| self.mark_pair_counts: Counter[str] = Counter() | |
| self.orphan_mark_counts: Counter[str] = Counter() | |
| self.warning_counts: Counter[str] = Counter() | |
| self.examples: dict[str, list[dict[str, Any]]] = defaultdict(list) | |
| def add_text(self, text: str, source_id: str) -> None: | |
| self.rows += 1 | |
| self.total_chars += len(text) | |
| if unicodedata.normalize("NFC", text) != text: | |
| self.nfc_changed_rows += 1 | |
| if unicodedata.normalize("NFD", text) != text: | |
| self.nfd_changed_rows += 1 | |
| row_chars = set() | |
| for ch in text: | |
| cls = _limbu_class(ch) | |
| if cls is None: | |
| continue | |
| self.limbu_chars += 1 | |
| self.char_counts[ch] += 1 | |
| self.class_counts[cls] += 1 | |
| row_chars.add(ch) | |
| for ch in row_chars: | |
| self.row_counts[ch] += 1 | |
| self._scan_clusters(text, source_id) | |
| def _scan_clusters(self, text: str, source_id: str) -> None: | |
| cluster_chars: list[str] = [] | |
| cluster_classes: list[str] = [] | |
| def flush() -> None: | |
| if not cluster_chars: | |
| return | |
| cluster_text = "".join(cluster_chars) | |
| record = _cluster_record(cluster_text, cluster_classes) | |
| record["source_id"] = source_id | |
| self.cluster_count += 1 | |
| self.cluster_signatures[record["signature"]] += 1 | |
| self.cluster_class_sets[_signature(sorted(set(cluster_classes), key=CLASS_ORDER.index))] += 1 | |
| for left, right in zip(cluster_classes, cluster_classes[1:]): | |
| if left != "base" or right != "base": | |
| self.mark_pair_counts[f"{left}+{right}"] += 1 | |
| for key in ( | |
| "has_duplicate_vowel", | |
| "has_duplicate_subjoined", | |
| "has_duplicate_final", | |
| "has_duplicate_sign", | |
| "has_mukphreng_after_final", | |
| "normalization_changed_nfc", | |
| "normalization_changed_nfd", | |
| ): | |
| if record[key]: | |
| self.warning_counts[key] += 1 | |
| _add_example(self.examples, key, record, self.example_limit) | |
| for ch in text: | |
| cls = _limbu_class(ch) | |
| if cls == "base": | |
| flush() | |
| cluster_chars = [ch] | |
| cluster_classes = [cls] | |
| elif cls in {"vowel", "subjoined", "final", "sign"}: | |
| if not cluster_chars: | |
| self.orphan_mark_counts[ch] += 1 | |
| self.warning_counts["orphan_mark"] += 1 | |
| _add_example( | |
| self.examples, | |
| "orphan_mark", | |
| { | |
| "source_id": source_id, | |
| "text": ch, | |
| "codepoints": f"U+{ord(ch):04X}", | |
| "names": unicodedata.name(ch), | |
| "class": cls, | |
| }, | |
| self.example_limit, | |
| ) | |
| else: | |
| cluster_chars.append(ch) | |
| cluster_classes.append(cls) | |
| else: | |
| flush() | |
| cluster_chars = [] | |
| cluster_classes = [] | |
| flush() | |
| def to_dict(self, sources: list[dict[str, Any]], top_n: int) -> dict[str, Any]: | |
| assigned = _assigned_limbu_chars() | |
| char_records = [_char_record(ch, self.char_counts[ch], self.row_counts[ch]) for ch in assigned] | |
| missing = [record for record in char_records if record["count"] == 0] | |
| underrepresented = { | |
| str(threshold): [record for record in char_records if record["count"] < threshold] | |
| for threshold in UNDERREPRESENTED_THRESHOLDS | |
| } | |
| return { | |
| "schema": SCHEMA, | |
| "unicode_data_version": unicodedata.unidata_version, | |
| "sources": sources, | |
| "rows": self.rows, | |
| "total_chars": self.total_chars, | |
| "limbu_chars": self.limbu_chars, | |
| "nfc_changed_rows": self.nfc_changed_rows, | |
| "nfd_changed_rows": self.nfd_changed_rows, | |
| "assigned_limbu_codepoints": len(assigned), | |
| "covered_limbu_codepoints": sum(1 for record in char_records if record["count"] > 0), | |
| "missing_limbu_codepoints": missing, | |
| "underrepresented_limbu_codepoints": underrepresented, | |
| "class_counts": dict(sorted(self.class_counts.items())), | |
| "char_counts": char_records, | |
| "diacritic_counts": [ | |
| record for record in char_records if record["class"] in {"vowel", "subjoined", "final", "sign"} | |
| ], | |
| "cluster_count": self.cluster_count, | |
| "cluster_signature_counts": self.cluster_signatures.most_common(top_n), | |
| "cluster_class_set_counts": self.cluster_class_sets.most_common(top_n), | |
| "mark_pair_counts": self.mark_pair_counts.most_common(top_n), | |
| "orphan_mark_counts": [ | |
| _char_record(ch, count, 0) for ch, count in sorted(self.orphan_mark_counts.items(), key=lambda item: ord(item[0])) | |
| ], | |
| "warning_counts": dict(sorted(self.warning_counts.items())), | |
| "warning_examples": self.examples, | |
| } | |
| def _iter_manifest_texts(path: Path) -> Iterable[tuple[str, str]]: | |
| for row_number, row in enumerate(read_jsonl(path), start=1): | |
| text = row.get("text") | |
| if not isinstance(text, str): | |
| raise DataValidationError(f"manifest row missing string text at {path}:{row_number}") | |
| source_id = str(row.get("sample_id") or f"{path}:{row_number}") | |
| yield source_id, text | |
| def _iter_label_texts(path: Path) -> Iterable[tuple[str, str]]: | |
| with path.open("r", encoding="utf-8") as handle: | |
| for row_number, raw in enumerate(handle, start=1): | |
| line = raw.rstrip("\n") | |
| if not line: | |
| continue | |
| _image_path, separator, text = line.partition("\t") | |
| if not separator: | |
| raise DataValidationError(f"label row without TAB at {path}:{row_number}") | |
| yield f"{path}:{row_number}", text | |
| def analyze(args: argparse.Namespace) -> dict[str, Any]: | |
| if not args.manifest and not args.label_file: | |
| raise DataValidationError("at least one --manifest or --label-file is required") | |
| stats = CorpusStats(example_limit=args.example_limit) | |
| sources: list[dict[str, Any]] = [] | |
| for value in args.manifest: | |
| path = Path(value) | |
| if not path.is_file(): | |
| raise DataValidationError(f"manifest does not exist: {path}") | |
| row_count = 0 | |
| for source_id, text in _iter_manifest_texts(path): | |
| stats.add_text(text, source_id) | |
| row_count += 1 | |
| sources.append({"type": "manifest", "path": str(path), "sha256": sha256_file(path), "rows": row_count}) | |
| for value in args.label_file: | |
| path = Path(value) | |
| if not path.is_file(): | |
| raise DataValidationError(f"label file does not exist: {path}") | |
| row_count = 0 | |
| for source_id, text in _iter_label_texts(path): | |
| stats.add_text(text, source_id) | |
| row_count += 1 | |
| sources.append({"type": "label_file", "path": str(path), "sha256": sha256_file(path), "rows": row_count}) | |
| summary = stats.to_dict(sources, args.top_n) | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| (out / "limbu-unicode-corpus-stats.json").write_text( | |
| json.dumps(summary, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| _write_markdown(out / "limbu-unicode-corpus-stats.md", summary) | |
| return summary | |
| def _write_markdown(path: Path, summary: dict[str, Any]) -> None: | |
| lines = [ | |
| "# Limbu Unicode Corpus Stats", | |
| "", | |
| f"- rows: `{summary['rows']}`", | |
| f"- Limbu chars: `{summary['limbu_chars']}`", | |
| f"- Unicode data version: `{summary['unicode_data_version']}`", | |
| f"- assigned Limbu codepoints: `{summary['assigned_limbu_codepoints']}`", | |
| f"- covered Limbu codepoints: `{summary['covered_limbu_codepoints']}`", | |
| f"- missing Limbu codepoints: `{len(summary['missing_limbu_codepoints'])}`", | |
| f"- NFC-changed rows: `{summary['nfc_changed_rows']}`", | |
| f"- NFD-changed rows: `{summary['nfd_changed_rows']}`", | |
| "", | |
| "## Missing Codepoints", | |
| "", | |
| ] | |
| if summary["missing_limbu_codepoints"]: | |
| for record in summary["missing_limbu_codepoints"]: | |
| lines.append(f"- `{record['codepoint']}` {record['char']} {record['name']}") | |
| else: | |
| lines.append("- none") | |
| lines.extend( | |
| [ | |
| "", | |
| "## Under 1000 Occurrences", | |
| "", | |
| ] | |
| ) | |
| under_1000 = summary["underrepresented_limbu_codepoints"]["1000"] | |
| if under_1000: | |
| for record in under_1000: | |
| lines.append(f"- `{record['codepoint']}` {record['char']} {record['name']}: `{record['count']}`") | |
| else: | |
| lines.append("- none") | |
| lines.extend( | |
| [ | |
| "", | |
| "## Top Cluster Signatures", | |
| "", | |
| "| Signature | Count |", | |
| "| --- | ---: |", | |
| ] | |
| ) | |
| for signature, count in summary["cluster_signature_counts"]: | |
| lines.append(f"| `{signature}` | {count} |") | |
| lines.extend( | |
| [ | |
| "", | |
| "## Top Mark Pairs", | |
| "", | |
| "| Pair | Count |", | |
| "| --- | ---: |", | |
| ] | |
| ) | |
| for pair, count in summary["mark_pair_counts"]: | |
| lines.append(f"| `{pair}` | {count} |") | |
| lines.extend( | |
| [ | |
| "", | |
| "## Warning Counts", | |
| "", | |
| ] | |
| ) | |
| if summary["warning_counts"]: | |
| for key, count in summary["warning_counts"].items(): | |
| lines.append(f"- `{key}`: `{count}`") | |
| else: | |
| lines.append("- none") | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def parse_args(argv: list[str] | None = None) -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--manifest", action="append", default=[]) | |
| parser.add_argument("--label-file", action="append", default=[]) | |
| parser.add_argument("--out", required=True) | |
| parser.add_argument("--top-n", type=int, default=50) | |
| parser.add_argument("--example-limit", type=int, default=12) | |
| return parser.parse_args(argv) | |
| def main(argv: list[str] | None = None) -> int: | |
| summary = analyze(parse_args(argv)) | |
| print(json.dumps(summary, ensure_ascii=False, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |