Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download src/ocrtech/training.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 77.5 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/training.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/src/ocrtech/training.py
-
curl -L -o training.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/training.py
77.5 kB
"""Training recipe generation for recognizer and corrector models."""
from __future__ import annotations
import json
import math
import random
import shlex
import shutil
import subprocess
import tarfile
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .admission import assert_recognizer_family_not_stopped
from .datasets import audit_recognizer_corpus
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, prepare_hf_correction_pairs, sha256_file, write_manifest
from .metrics import cer, exact_line_accuracy, wer
from .models import audit_model_card, write_model_card
from .normalization import dictionary_from_texts, normalize_ocr_text
PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED = (
"https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/"
"devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams"
)
def write_recognizer_recipe(
train_manifest: str | Path,
eval_manifest: str | Path,
output_dir: str | Path,
*,
base_model: str = PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
dictionary_path: str | Path | None = None,
train_batch_size: int = 64,
eval_batch_size: int = 64,
learning_rate: float = 0.001,
epochs: float | None = None,
failure_review: str | Path | None = None,
run: bool = False,
) -> Path:
return write_paddle_recognizer_recipe(
train_manifest,
eval_manifest,
output_dir,
base_model=base_model,
dictionary_path=dictionary_path,
train_batch_size=train_batch_size,
eval_batch_size=eval_batch_size,
learning_rate=learning_rate,
epochs=epochs,
failure_review=failure_review,
run=run,
)
def write_paddle_recognizer_recipe(
train_manifest: str | Path,
eval_manifest: str | Path,
output_dir: str | Path,
*,
base_model: str = PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
dictionary_path: str | Path | None = None,
train_batch_size: int = 64,
eval_batch_size: int = 64,
learning_rate: float = 0.001,
epochs: float | None = None,
failure_review: str | Path | None = None,
run: bool = False,
) -> Path:
if train_batch_size <= 0:
raise DataValidationError("train_batch_size must be positive")
if eval_batch_size <= 0:
raise DataValidationError("eval_batch_size must be positive")
if learning_rate <= 0:
raise DataValidationError("learning_rate must be positive")
if epochs is not None and epochs <= 0:
raise DataValidationError("epochs must be positive when provided")
if failure_review is not None:
assert_recognizer_family_not_stopped(failure_review, backend="paddleocr", base_model=base_model)
train_entries = load_manifest(train_manifest)
eval_entries = load_manifest(eval_manifest)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
labels_dir = out / "labels"
labels_dir.mkdir(exist_ok=True)
train_label = labels_dir / "train.txt"
eval_label = labels_dir / "eval.txt"
_write_paddle_label_file(train_entries, train_label)
_write_paddle_label_file(eval_entries, eval_label)
dict_path = Path(dictionary_path) if dictionary_path else out / "nep_eng_dict.txt"
if dictionary_path is None:
chars = dictionary_from_texts([entry.text for entry in [*train_entries, *eval_entries]])
dict_path.write_text("\n".join(chars) + "\n", encoding="utf-8")
elif not dict_path.exists():
raise DataValidationError(f"dictionary_path does not exist: {dict_path}")
dictionary_chars = [line for line in dict_path.read_text(encoding="utf-8").splitlines() if line]
_validate_paddle_dictionary_coverage([*train_entries, *eval_entries], dictionary_chars, dict_path)
config_path = out / "paddle_recognizer.yml"
config_path.write_text(
_recognizer_config_text(
train_label=train_label,
eval_label=eval_label,
dictionary_path=dict_path,
base_model=base_model,
output_dir=out / "checkpoints",
train_batch_size=train_batch_size,
eval_batch_size=eval_batch_size,
learning_rate=learning_rate,
epochs=epochs,
),
encoding="utf-8",
)
summary_path = out / "recognizer-training-summary.json"
summary_path.write_text(
json.dumps(
_recognizer_training_summary(
train_manifest=Path(train_manifest),
eval_manifest=Path(eval_manifest),
train_entries=train_entries,
eval_entries=eval_entries,
train_label=train_label,
eval_label=eval_label,
dictionary_path=dict_path,
dictionary_chars=dictionary_chars,
config_path=config_path,
base_model=base_model,
train_batch_size=train_batch_size,
eval_batch_size=eval_batch_size,
learning_rate=learning_rate,
epochs=epochs,
),
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
(out / "README.md").write_text(
"\n".join(
[
"# Recognizer Training Recipe",
"",
f"Training summary: `{summary_path}`",
f"Base model: `{base_model}`",
f"Train labels: `{train_label}`",
f"Eval labels: `{eval_label}`",
f"Dictionary: `{dict_path}`",
f"Train batch size: `{train_batch_size}`",
f"Eval batch size: `{eval_batch_size}`",
f"Learning rate: `{learning_rate}`",
f"Epochs: `{epochs if epochs is not None else 'PaddleOCR default'}`",
"",
"Run with a checked-out PaddleOCR repo:",
"",
"```bash",
f"python PaddleOCR/tools/train.py -c {config_path}",
"```",
"",
]
),
encoding="utf-8",
)
if run:
completed = subprocess.run(["python", "PaddleOCR/tools/train.py", "-c", str(config_path)], check=False)
if completed.returncode != 0:
raise DataValidationError(f"PaddleOCR training failed with exit code {completed.returncode}")
return config_path
def _recognizer_training_summary(
*,
train_manifest: Path,
eval_manifest: Path,
train_entries: list[ManifestEntry],
eval_entries: list[ManifestEntry],
train_label: Path,
eval_label: Path,
dictionary_path: Path,
dictionary_chars: list[str],
config_path: Path,
base_model: str,
train_batch_size: int,
eval_batch_size: int,
learning_rate: float,
epochs: float | None,
) -> dict[str, Any]:
return {
"backend": "paddleocr",
"base_model": base_model,
"manifests": {
"train": _manifest_summary(train_manifest, train_entries),
"eval": _manifest_summary(eval_manifest, eval_entries),
},
"artifacts": {
"config": _file_summary(config_path),
"train_label": _file_summary(train_label),
"eval_label": _file_summary(eval_label),
"dictionary": {
**_file_summary(dictionary_path),
"character_count": len(dictionary_chars),
"contains_space": " " in dictionary_chars,
"contains_ascii": any(char.isascii() and char.isalnum() for char in dictionary_chars),
"contains_devanagari": any("\u0900" <= char <= "\u097f" for char in dictionary_chars),
},
},
"hyperparameters": {
"train_batch_size": train_batch_size,
"eval_batch_size": eval_batch_size,
"learning_rate": learning_rate,
"epochs": epochs,
},
}
def _manifest_summary(path: Path, entries: list[ManifestEntry]) -> dict[str, Any]:
texts = [entry.text for entry in entries]
chars = dictionary_from_texts(texts)
return {
"path": str(path),
"sha256": sha256_file(path) if path.exists() else None,
"sample_count": len(entries),
"dataset_counts": _count_values(entry.dataset for entry in entries),
"slice_counts": _slice_counts(entries),
"character_count": len(chars),
"contains_ascii": any(char.isascii() and char.isalnum() for char in chars),
"contains_devanagari": any("\u0900" <= char <= "\u097f" for char in chars),
}
def _file_summary(path: Path) -> dict[str, Any]:
return {
"path": str(path),
"sha256": sha256_file(path) if path.exists() and path.is_file() else None,
}
def _slice_counts(entries: list[ManifestEntry]) -> dict[str, int]:
counts: dict[str, int] = {}
for entry in entries:
for slice_name in _entry_slices(entry):
counts[slice_name] = counts.get(slice_name, 0) + 1
return dict(sorted(counts.items()))
def _entry_slices(entry: ManifestEntry) -> list[str]:
metadata = entry.metadata or {}
values: set[str] = set()
for key in ("slice", "script", "document_type", "language"):
value = metadata.get(key)
if isinstance(value, str) and value:
values.add(value)
for key in ("slices", "scripts", "tags"):
value = metadata.get(key)
if isinstance(value, list):
values.update(str(item) for item in value if str(item))
elif isinstance(value, str) and value:
values.add(value)
return sorted(values)
def _count_values(values: object) -> dict[str, int]:
counts: dict[str, int] = {}
for value in values:
key = str(value)
counts[key] = counts.get(key, 0) + 1
return dict(sorted(counts.items()))
def _write_paddle_label_file(entries: list[ManifestEntry], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
for entry in entries:
if "\t" in entry.image_path or "\n" in entry.image_path or "\r" in entry.image_path:
raise DataValidationError(f"image_path contains a tab or newline and cannot be serialized for PaddleOCR: {entry.sample_id}")
handle.write(f"{entry.image_path}\t{_paddle_label_text(entry.text)}\n")
def _validate_paddle_dictionary_coverage(
entries: list[ManifestEntry],
dictionary_chars: list[str],
dictionary_path: Path,
) -> None:
allowed = set(dictionary_chars)
missing: dict[str, list[str]] = {}
for entry in entries:
label = _paddle_label_text(entry.text)
missing_chars = sorted({char for char in label if char not in allowed})
if missing_chars:
missing[entry.sample_id] = missing_chars
if missing:
details = [
f"{sample_id}: {''.join(chars)}"
for sample_id, chars in list(missing.items())[:10]
]
suffix = "; ".join(details)
raise DataValidationError(
f"dictionary_path is missing label characters for {len(missing)} samples: {dictionary_path}; {suffix}"
)
def _paddle_label_text(text: str) -> str:
label = normalize_ocr_text(text).replace("\t", " ")
return " ".join(label.split())
def _yaml_float(value: float) -> str:
if not math.isfinite(value):
raise DataValidationError(f"PaddleOCR YAML float must be finite: {value!r}")
if value == 0:
return "0.0"
text = f"{value:.12f}".rstrip("0").rstrip(".")
if "." not in text:
text += ".0"
return text
def _recognizer_config_text(
*,
train_label: Path,
eval_label: Path,
dictionary_path: Path,
base_model: str,
output_dir: Path,
train_batch_size: int,
eval_batch_size: int,
learning_rate: float,
epochs: float | None,
) -> str:
epoch_value: int | float | None
if epochs is None:
epoch_value = None
elif float(epochs).is_integer():
epoch_value = int(epochs)
else:
epoch_value = epochs
epoch_line = f" epoch_num: {epoch_value}\n" if epoch_value is not None else ""
learning_rate_value = _yaml_float(float(learning_rate))
return f"""# Generated by ocrtech train-recognizer.
# PP-OCRv3 mobile recognizer recipe for PaddleOCR's tools/train.py.
# Keep this in sync with PaddleOCR configs/rec/PP-OCRv3/multi_language/devanagari_PP-OCRv3_mobile_rec.yml.
Global:
model_name: devanagari_PP-OCRv3_mobile_rec
debug: false
use_gpu: true
{epoch_line} log_smooth_window: 20
print_batch_step: 10
pretrained_model: {base_model}
character_dict_path: {dictionary_path}
save_model_dir: {output_dir}
save_epoch_step: 1
eval_batch_step:
- 0
- 200
cal_metric_during_train: true
use_space_char: true
max_text_length: 128
infer_mode: false
distributed: false
save_res_path: {output_dir.parent / "predicts.txt"}
Optimizer:
name: Adam
beta1: 0.9
beta2: 0.999
lr:
name: Cosine
learning_rate: {learning_rate_value}
warmup_epoch: 1
regularizer:
name: L2
factor: 3.0e-05
Architecture:
model_type: rec
algorithm: SVTR_LCNet
Transform:
Backbone:
name: MobileNetV1Enhance
scale: 0.5
last_conv_stride:
- 1
- 2
last_pool_type: avg
last_pool_kernel_size:
- 2
- 2
Head:
name: MultiHead
head_list:
- CTCHead:
Neck:
name: svtr
dims: 64
depth: 2
hidden_dims: 120
use_guide: true
Head:
fc_decay: 1.0e-05
- SARHead:
enc_dim: 512
max_text_length: 128
Loss:
name: MultiLoss
loss_config_list:
- CTCLoss:
- SARLoss:
PostProcess:
name: CTCLabelDecode
Metric:
name: RecMetric
main_indicator: acc
ignore_space: false
Train:
dataset:
name: SimpleDataSet
data_dir: .
ext_op_transform_idx: 1
label_file_list:
- {train_label}
transforms:
- DecodeImage:
img_mode: BGR
channel_first: false
- RecConAug:
prob: 0.5
ext_data_num: 2
image_shape:
- 48
- 320
- 3
- RecAug:
- MultiLabelEncode:
- RecResizeImg:
image_shape:
- 3
- 48
- 320
- KeepKeys:
keep_keys:
- image
- label_ctc
- label_sar
- length
- valid_ratio
loader:
shuffle: true
batch_size_per_card: {train_batch_size}
drop_last: true
num_workers: 2
Eval:
dataset:
name: SimpleDataSet
data_dir: .
label_file_list:
- {eval_label}
transforms:
- DecodeImage:
img_mode: BGR
channel_first: false
- MultiLabelEncode:
- RecResizeImg:
image_shape:
- 3
- 48
- 320
- KeepKeys:
keep_keys:
- image
- label_ctc
- label_sar
- length
- valid_ratio
loader:
shuffle: false
batch_size_per_card: {eval_batch_size}
drop_last: false
num_workers: 2
"""
@dataclass(slots=True)
class HfRecognizerTrainingConfig:
train_manifest: str
eval_manifest: str
output_dir: str
base_model: str
max_target_length: int
per_device_train_batch_size: int
per_device_eval_batch_size: int
learning_rate: float
num_train_epochs: float
warmup_steps: int
logging_steps: int
save_total_limit: int
predict_with_generate: bool
generation_max_length: int
def to_dict(self) -> dict[str, Any]:
return {
"train_manifest": self.train_manifest,
"eval_manifest": self.eval_manifest,
"output_dir": self.output_dir,
"base_model": self.base_model,
"max_target_length": self.max_target_length,
"per_device_train_batch_size": self.per_device_train_batch_size,
"per_device_eval_batch_size": self.per_device_eval_batch_size,
"learning_rate": self.learning_rate,
"num_train_epochs": self.num_train_epochs,
"warmup_steps": self.warmup_steps,
"logging_steps": self.logging_steps,
"save_total_limit": self.save_total_limit,
"predict_with_generate": self.predict_with_generate,
"generation_max_length": self.generation_max_length,
}
@classmethod
def from_path(cls, path: str | Path) -> "HfRecognizerTrainingConfig":
payload = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise DataValidationError("hf recognizer config must be a JSON object")
return cls(
train_manifest=str(payload.get("train_manifest") or ""),
eval_manifest=str(payload.get("eval_manifest") or ""),
output_dir=str(payload.get("output_dir") or ""),
base_model=str(payload.get("base_model") or ""),
max_target_length=int(payload.get("max_target_length", 128)),
per_device_train_batch_size=int(payload.get("per_device_train_batch_size", 8)),
per_device_eval_batch_size=int(payload.get("per_device_eval_batch_size", 8)),
learning_rate=float(payload.get("learning_rate", 5e-5)),
num_train_epochs=float(payload.get("num_train_epochs", 3.0)),
warmup_steps=int(payload.get("warmup_steps", 0)),
logging_steps=int(payload.get("logging_steps", 10)),
save_total_limit=int(payload.get("save_total_limit", 2)),
predict_with_generate=bool(payload.get("predict_with_generate", True)),
generation_max_length=int(payload.get("generation_max_length", payload.get("max_target_length", 128))),
)
@dataclass(slots=True)
class HfTextCorrectorTrainingConfig:
train_pairs: str
eval_pairs: str
output_dir: str
base_model: str
max_source_length: int
max_target_length: int
per_device_train_batch_size: int
per_device_eval_batch_size: int
learning_rate: float
num_train_epochs: float
warmup_steps: int
logging_steps: int
save_total_limit: int
predict_with_generate: bool
generation_max_length: int
def to_dict(self) -> dict[str, Any]:
return {
"train_pairs": self.train_pairs,
"eval_pairs": self.eval_pairs,
"output_dir": self.output_dir,
"base_model": self.base_model,
"max_source_length": self.max_source_length,
"max_target_length": self.max_target_length,
"per_device_train_batch_size": self.per_device_train_batch_size,
"per_device_eval_batch_size": self.per_device_eval_batch_size,
"learning_rate": self.learning_rate,
"num_train_epochs": self.num_train_epochs,
"warmup_steps": self.warmup_steps,
"logging_steps": self.logging_steps,
"save_total_limit": self.save_total_limit,
"predict_with_generate": self.predict_with_generate,
"generation_max_length": self.generation_max_length,
}
@classmethod
def from_path(cls, path: str | Path) -> "HfTextCorrectorTrainingConfig":
payload = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise DataValidationError("hf text corrector config must be a JSON object")
return cls(
train_pairs=str(payload.get("train_pairs") or ""),
eval_pairs=str(payload.get("eval_pairs") or ""),
output_dir=str(payload.get("output_dir") or ""),
base_model=str(payload.get("base_model") or ""),
max_source_length=int(payload.get("max_source_length", 256)),
max_target_length=int(payload.get("max_target_length", 256)),
per_device_train_batch_size=int(payload.get("per_device_train_batch_size", 16)),
per_device_eval_batch_size=int(payload.get("per_device_eval_batch_size", 16)),
learning_rate=float(payload.get("learning_rate", 5e-5)),
num_train_epochs=float(payload.get("num_train_epochs", 3.0)),
warmup_steps=int(payload.get("warmup_steps", 0)),
logging_steps=int(payload.get("logging_steps", 10)),
save_total_limit=int(payload.get("save_total_limit", 2)),
predict_with_generate=bool(payload.get("predict_with_generate", True)),
generation_max_length=int(payload.get("generation_max_length", payload.get("max_target_length", 256))),
)
@dataclass(slots=True)
class RecognizerTrainingBundle:
archive_path: str
manifest_path: str
readme_path: str
file_count: int
total_bytes: int
def to_dict(self) -> dict[str, Any]:
return {
"archive_path": self.archive_path,
"manifest_path": self.manifest_path,
"readme_path": self.readme_path,
"file_count": self.file_count,
"total_bytes": self.total_bytes,
}
@dataclass(slots=True)
class HfRecognizerFinalization:
run_dir: str
package_dir: str
model_card_path: str
audit_path: str
report_json_path: str
report_md_path: str
audit_passed: bool
def to_dict(self) -> dict[str, Any]:
return {
"run_dir": self.run_dir,
"package_dir": self.package_dir,
"model_card_path": self.model_card_path,
"audit_path": self.audit_path,
"report_json_path": self.report_json_path,
"report_md_path": self.report_md_path,
"audit_passed": self.audit_passed,
}
def write_hf_recognizer_recipe(
train_manifest: str | Path,
eval_manifest: str | Path,
output_dir: str | Path,
*,
base_model: str = "microsoft/trocr-small-printed",
max_target_length: int = 128,
per_device_train_batch_size: int = 8,
per_device_eval_batch_size: int = 8,
learning_rate: float = 5e-5,
num_train_epochs: float = 3.0,
warmup_steps: int = 0,
logging_steps: int = 10,
save_total_limit: int = 2,
run: bool = False,
) -> Path:
train_entries = load_manifest(train_manifest)
eval_entries = load_manifest(eval_manifest)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
config = HfRecognizerTrainingConfig(
train_manifest=str(Path(train_manifest)),
eval_manifest=str(Path(eval_manifest)),
output_dir=str(out),
base_model=base_model,
max_target_length=max_target_length,
per_device_train_batch_size=per_device_train_batch_size,
per_device_eval_batch_size=per_device_eval_batch_size,
learning_rate=learning_rate,
num_train_epochs=num_train_epochs,
warmup_steps=warmup_steps,
logging_steps=logging_steps,
save_total_limit=save_total_limit,
predict_with_generate=True,
generation_max_length=max_target_length,
)
config_path = out / "hf_recognizer_config.json"
config_path.write_text(json.dumps(config.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
summary_path = out / "recognizer-training-summary.json"
summary_path.write_text(
json.dumps(
_hf_recognizer_training_summary(
train_manifest=Path(train_manifest),
eval_manifest=Path(eval_manifest),
train_entries=train_entries,
eval_entries=eval_entries,
config_path=config_path,
base_model=base_model,
max_target_length=max_target_length,
per_device_train_batch_size=per_device_train_batch_size,
per_device_eval_batch_size=per_device_eval_batch_size,
learning_rate=learning_rate,
num_train_epochs=num_train_epochs,
warmup_steps=warmup_steps,
logging_steps=logging_steps,
save_total_limit=save_total_limit,
),
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
(out / "README.md").write_text(
"\n".join(
[
"# Hugging Face Recognizer Training Recipe",
"",
f"Training summary: `{summary_path}`",
f"Base model: `{base_model}`",
f"Train manifest: `{train_manifest}` ({len(train_entries)} samples)",
f"Eval manifest: `{eval_manifest}` ({len(eval_entries)} samples)",
"",
"Install the backend dependencies first:",
"",
"```bash",
"python -m pip install -e '.[hf-recognizer]'",
"```",
"",
"Run in-process:",
"",
"```bash",
_hf_recognizer_train_command(config, run=True),
"```",
]
)
+ "\n",
encoding="utf-8",
)
if run:
run_hf_recognizer_training(config_path)
return config_path
def _hf_recognizer_train_command(config: HfRecognizerTrainingConfig, *, run: bool = False) -> str:
parts = [
"ocrtech",
"train-recognizer",
"--backend",
"hf-vision-encoder-decoder",
"--base-model",
config.base_model,
"--train-manifest",
config.train_manifest,
"--eval-manifest",
config.eval_manifest,
"--out",
config.output_dir,
"--max-target-length",
str(config.max_target_length),
"--train-batch-size",
str(config.per_device_train_batch_size),
"--eval-batch-size",
str(config.per_device_eval_batch_size),
"--learning-rate",
_format_float(config.learning_rate),
"--epochs",
_format_float(config.num_train_epochs),
]
if run:
parts.append("--run")
return " ".join(shlex.quote(part) for part in parts)
def _format_float(value: float) -> str:
return f"{value:.12g}"
def _hf_recognizer_training_summary(
*,
train_manifest: Path,
eval_manifest: Path,
train_entries: list[ManifestEntry],
eval_entries: list[ManifestEntry],
config_path: Path,
base_model: str,
max_target_length: int,
per_device_train_batch_size: int,
per_device_eval_batch_size: int,
learning_rate: float,
num_train_epochs: float,
warmup_steps: int,
logging_steps: int,
save_total_limit: int,
) -> dict[str, Any]:
texts = [entry.text for entry in [*train_entries, *eval_entries]]
chars = dictionary_from_texts(texts)
return {
"backend": "hf_vision_encoder_decoder",
"base_model": base_model,
"manifests": {
"train": _manifest_summary(train_manifest, train_entries),
"eval": _manifest_summary(eval_manifest, eval_entries),
},
"artifacts": {
"config": _file_summary(config_path),
"dictionary": {
"character_count": len(chars),
"contains_space": " " in chars,
"contains_ascii": any(char.isascii() and char.isalnum() for char in chars),
"contains_devanagari": any("\u0900" <= char <= "\u097f" for char in chars),
},
},
"hyperparameters": {
"max_target_length": max_target_length,
"per_device_train_batch_size": per_device_train_batch_size,
"per_device_eval_batch_size": per_device_eval_batch_size,
"learning_rate": learning_rate,
"num_train_epochs": num_train_epochs,
"warmup_steps": warmup_steps,
"logging_steps": logging_steps,
"save_total_limit": save_total_limit,
"generation_max_length": max_target_length,
},
}
def run_hf_recognizer_training(config_path: str | Path) -> Path:
config = HfRecognizerTrainingConfig.from_path(config_path)
train_entries = load_manifest(config.train_manifest)
eval_entries = load_manifest(config.eval_manifest)
if not train_entries:
raise DataValidationError("hf recognizer training requires non-empty train_manifest")
if not eval_entries:
raise DataValidationError("hf recognizer training requires non-empty eval_manifest")
try:
import torch
except ImportError as exc:
raise DataValidationError("hf recognizer training requires torch. Install ocr-tech[hf-recognizer].") from exc
try:
from PIL import Image
except ImportError as exc:
raise DataValidationError("hf recognizer training requires Pillow. Install ocr-tech[hf-recognizer].") from exc
try:
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments, TrOCRProcessor, VisionEncoderDecoderModel
except ImportError as exc:
raise DataValidationError("hf recognizer training requires transformers. Install ocr-tech[hf-recognizer].") from exc
output_dir = Path(config.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
processor = _load_trocr_processor(TrOCRProcessor, config.base_model)
model = VisionEncoderDecoderModel.from_pretrained(config.base_model)
tokenizer = processor.tokenizer
decoder_start_token_id = _read_token_id(model.config, "decoder_start_token_id")
pad_token_id = _read_token_id(model.config, "pad_token_id")
if decoder_start_token_id is None:
decoder_start_token_id = tokenizer.cls_token_id or tokenizer.bos_token_id
if pad_token_id is None:
pad_token_id = tokenizer.pad_token_id
if decoder_start_token_id is None or pad_token_id is None:
raise DataValidationError("hf recognizer base model tokenizer must define decoder_start_token_id and pad_token_id")
_write_token_ids(model, decoder_start_token_id=decoder_start_token_id, pad_token_id=pad_token_id)
model.generation_config.max_length = config.generation_max_length
train_dataset = _VisionTextDataset(train_entries, processor, Image, config.max_target_length)
eval_dataset = _VisionTextDataset(eval_entries, processor, Image, config.max_target_length)
training_args = Seq2SeqTrainingArguments(
output_dir=str(output_dir / "checkpoints"),
eval_strategy="epoch",
save_strategy="epoch",
learning_rate=config.learning_rate,
per_device_train_batch_size=config.per_device_train_batch_size,
per_device_eval_batch_size=config.per_device_eval_batch_size,
num_train_epochs=config.num_train_epochs,
warmup_steps=config.warmup_steps,
logging_steps=config.logging_steps,
save_total_limit=config.save_total_limit,
predict_with_generate=config.predict_with_generate,
remove_unused_columns=False,
report_to=[],
)
trainer = Seq2SeqTrainer(
model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
processing_class=processor,
data_collator=_VisionTextCollator(processor, torch),
compute_metrics=_hf_compute_metrics_factory(processor),
)
trainer.train()
final_model_dir = output_dir / "final-model"
trainer.save_model(str(final_model_dir))
processor.save_pretrained(str(final_model_dir))
metrics = trainer.evaluate(max_length=config.generation_max_length)
metrics_path = output_dir / "training_metrics.json"
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return final_model_dir
def bundle_recognizer_training(
recipe_dir: str | Path,
output_dir: str | Path,
*,
train_manifest: str | Path,
eval_manifest: str | Path,
archive_name: str = "recognizer-training-bundle.tar.gz",
base_dir: str | Path = ".",
require_corpus_audit: bool = False,
min_train_samples: int = 1000,
min_eval_samples: int = 100,
min_train_english: int = 200,
min_train_devanagari: int = 200,
min_train_mixed: int = 25,
min_eval_english: int = 50,
min_eval_devanagari: int = 50,
min_eval_mixed: int = 10,
min_train_latin_only: int = 50,
min_train_devanagari_only: int = 50,
min_eval_latin_only: int = 10,
min_eval_devanagari_only: int = 10,
min_train_real: int = 100,
min_train_synthetic: int = 100,
failure_review: str | Path | None = None,
) -> RecognizerTrainingBundle:
base = Path(base_dir).resolve()
recipe_path = _resolve_under_base(recipe_dir, base=base)
train_path = _resolve_under_base(train_manifest, base=base)
eval_path = _resolve_under_base(eval_manifest, base=base)
if not recipe_path.is_dir():
raise DataValidationError(f"recipe_dir does not exist or is not a directory: {recipe_path}")
if not train_path.is_file():
raise DataValidationError(f"train_manifest does not exist or is not a file: {train_path}")
if not eval_path.is_file():
raise DataValidationError(f"eval_manifest does not exist or is not a file: {eval_path}")
training_summary_path = recipe_path / "recognizer-training-summary.json"
if failure_review is not None:
if not training_summary_path.is_file():
raise DataValidationError(f"recognizer training summary does not exist for stopped-family guard: {training_summary_path}")
summary_payload = _read_json_object(training_summary_path, "recognizer training summary")
assert_recognizer_family_not_stopped(
failure_review,
backend=str(summary_payload.get("backend") or ""),
base_model=str(summary_payload.get("base_model") or ""),
)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
corpus_audit_payload: dict[str, Any] = {"required": require_corpus_audit, "present": False}
if require_corpus_audit:
corpus_audit = audit_recognizer_corpus(
train_path,
eval_path,
out / "recognizer-corpus-audit",
min_train_samples=min_train_samples,
min_eval_samples=min_eval_samples,
min_train_english=min_train_english,
min_train_devanagari=min_train_devanagari,
min_train_mixed=min_train_mixed,
min_eval_english=min_eval_english,
min_eval_devanagari=min_eval_devanagari,
min_eval_mixed=min_eval_mixed,
min_train_latin_only=min_train_latin_only,
min_train_devanagari_only=min_train_devanagari_only,
min_eval_latin_only=min_eval_latin_only,
min_eval_devanagari_only=min_eval_devanagari_only,
min_train_real=min_train_real,
min_train_synthetic=min_train_synthetic,
)
corpus_audit_payload = {
"required": True,
"present": True,
"passed": corpus_audit.passed,
"path": "recognizer-corpus-audit/recognizer-corpus-audit.json",
}
if not corpus_audit.passed:
raise DataValidationError(
"recognizer corpus audit failed; refusing to bundle training job: "
+ "; ".join(corpus_audit.issues)
)
archive_path = out / archive_name
manifest_path = out / "recognizer-training-bundle.json"
readme_path = out / "recognizer-training-bundle.md"
train_entries = load_manifest(train_path)
eval_entries = load_manifest(eval_path)
with tempfile.TemporaryDirectory(prefix="ocrtech-recognizer-bundle-") as tmp_dir_name:
staging = Path(tmp_dir_name)
recipe_arcname = _relative_to_base(recipe_path, base=base)
train_arcname = _relative_to_base(train_path, base=base)
eval_arcname = _relative_to_base(eval_path, base=base)
shutil.copytree(recipe_path, staging / recipe_arcname)
rewritten_train = _stage_bundle_manifest(train_entries, staging, split_name="train", base=base)
rewritten_eval = _stage_bundle_manifest(eval_entries, staging, split_name="eval", base=base)
write_manifest(rewritten_train, staging / train_arcname)
write_manifest(rewritten_eval, staging / eval_arcname)
_write_paddle_label_file(rewritten_train, staging / recipe_arcname / "labels" / "train.txt")
_write_paddle_label_file(rewritten_eval, staging / recipe_arcname / "labels" / "eval.txt")
staged_files = sorted(path for path in staging.rglob("*") if path.is_file())
total_bytes = sum(path.stat().st_size for path in staged_files)
staged_summary = staging / recipe_arcname / "recognizer-training-summary.json"
summary_payload = _bundle_file_reference(staged_summary, staging)
recipe_backend, run_command = _bundle_recipe_run_command(staging / recipe_arcname, recipe_arcname=recipe_arcname)
with tarfile.open(archive_path, "w:gz") as archive:
for source in staged_files:
archive.add(source, arcname=str(source.relative_to(staging)))
payload = {
"archive_path": str(archive_path),
"base_dir": str(base),
"recipe_dir": str(_relative_to_base(recipe_path, base=base)),
"train_manifest": str(_relative_to_base(train_path, base=base)),
"eval_manifest": str(_relative_to_base(eval_path, base=base)),
"portable_image_root": "bundle_assets/images",
"recipe_backend": recipe_backend,
"run_command": run_command,
"training_summary": summary_payload,
"recognizer_corpus_audit": corpus_audit_payload,
"file_count": len(staged_files),
"total_bytes": total_bytes,
"files": [str(path.relative_to(staging)) for path in staged_files],
}
manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
readme_path.write_text(
"# Recognizer Training Bundle\n\n"
f"Archive: `{archive_path}`\n"
f"Training summary: `{summary_payload.get('path') or 'not present'}`\n"
f"Recognizer corpus audit: `{corpus_audit_payload.get('path') or 'not required'}`\n"
f"Files: `{len(staged_files)}`\n"
f"Bytes before compression: `{total_bytes}`\n\n"
"Extract this archive at the root of an ocr-tech workspace or an empty training workspace. "
"The recipe uses relative paths, so run training from the extraction root.\n\n"
"```bash\n"
f"tar -xzf {archive_path.name}\n"
f"{run_command}\n"
"```\n",
encoding="utf-8",
)
return RecognizerTrainingBundle(
archive_path=str(archive_path),
manifest_path=str(manifest_path),
readme_path=str(readme_path),
file_count=len(staged_files),
total_bytes=total_bytes,
)
def _bundle_recipe_run_command(recipe_dir: Path, *, recipe_arcname: Path) -> tuple[str, str]:
paddle_config = recipe_dir / "paddle_recognizer.yml"
hf_config = recipe_dir / "hf_recognizer_config.json"
if paddle_config.is_file():
return "paddleocr", f"python PaddleOCR/tools/train.py -c {recipe_arcname / 'paddle_recognizer.yml'}"
if hf_config.is_file():
config = HfRecognizerTrainingConfig.from_path(hf_config)
return "hf_vision_encoder_decoder", _hf_recognizer_train_command(config, run=True)
return "unknown", f"# no supported recognizer training config found under {recipe_dir}"
def _bundle_file_reference(path: Path, staging_root: Path) -> dict[str, Any]:
if not path.exists() or not path.is_file():
return {"present": False, "path": str(path.relative_to(staging_root))}
return {
"present": True,
"path": str(path.relative_to(staging_root)),
"sha256": sha256_file(path),
}
def _read_json_object(path: Path, label: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise DataValidationError(f"{label} does not exist: {path}") from exc
except json.JSONDecodeError as exc:
raise DataValidationError(f"invalid {label} JSON {path}: {exc}") from exc
if not isinstance(payload, dict):
raise DataValidationError(f"{label} must be a JSON object: {path}")
return payload
def _stage_bundle_manifest(entries: list[ManifestEntry], staging: Path, *, split_name: str, base: Path) -> list[ManifestEntry]:
rewritten: list[ManifestEntry] = []
for entry in entries:
image_path = Path(entry.image_path)
if not image_path.is_absolute():
image_path = base / image_path
image_path = image_path.resolve()
if not image_path.exists():
raise DataValidationError(f"manifest image_path does not exist for {entry.sample_id}: {image_path}")
if not image_path.is_file():
raise DataValidationError(f"recognizer training bundle only supports file image_path entries: {entry.sample_id}: {image_path}")
suffix = image_path.suffix or ".img"
target_rel = Path("bundle_assets") / "images" / split_name / f"{_safe_sample_id(entry.sample_id)}{suffix}"
target = staging / target_rel
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(image_path, target)
rewritten.append(
ManifestEntry(
sample_id=entry.sample_id,
dataset=entry.dataset,
split=entry.split,
image_path=str(target_rel),
text=entry.text,
sha256=entry.sha256,
metadata=dict(entry.metadata),
)
)
return rewritten
def _safe_sample_id(sample_id: str) -> str:
safe = "".join(char if char.isalnum() or char in {"-", "_", "."} else "_" for char in sample_id)
return safe or "sample"
def _resolve_under_base(path: str | Path, *, base: Path) -> Path:
value = Path(path)
resolved = value if value.is_absolute() else base / value
resolved = resolved.resolve()
try:
resolved.relative_to(base)
except ValueError as exc:
raise DataValidationError(f"path must be inside base_dir {base}: {path}") from exc
return resolved
def _relative_to_base(path: Path, *, base: Path) -> Path:
try:
return path.resolve().relative_to(base)
except ValueError as exc:
raise DataValidationError(f"path must be inside base_dir {base}: {path}") from exc
def _load_trocr_processor(processor_cls: type[object], source: str | Path) -> object:
attempts = ({"backend": "pil"}, {"use_fast": False})
for kwargs in attempts:
try:
return processor_cls.from_pretrained(str(source), **kwargs)
except TypeError as exc:
if any(key in str(exc) for key in kwargs):
continue
raise
except ImportError as exc:
raise DataValidationError("TrOCR processor loading requires sentencepiece and protobuf. Install ocr-tech[hf-recognizer].") from exc
except ValueError as exc:
message = str(exc)
if "SentencePiece" in message or "sentencepiece" in message or "tiktoken" in message or "protobuf" in message:
raise DataValidationError("TrOCR processor loading requires sentencepiece and protobuf. Install ocr-tech[hf-recognizer].") from exc
raise
return processor_cls.from_pretrained(str(source))
def _read_token_id(config: object, name: str) -> int | None:
value = getattr(config, name, None)
if value is not None:
return int(value)
nested_decoder = getattr(config, "decoder", None)
nested_value = getattr(nested_decoder, name, None)
return int(nested_value) if nested_value is not None else None
def _set_attr_if_possible(target: object, name: str, value: int) -> None:
try:
setattr(target, name, value)
except AttributeError:
return
def _write_token_ids(model: object, *, decoder_start_token_id: int, pad_token_id: int) -> None:
config = getattr(model, "config", None)
if config is not None:
_set_attr_if_possible(config, "decoder_start_token_id", decoder_start_token_id)
_set_attr_if_possible(config, "pad_token_id", pad_token_id)
nested_decoder = getattr(config, "decoder", None)
if nested_decoder is not None:
_set_attr_if_possible(nested_decoder, "decoder_start_token_id", decoder_start_token_id)
_set_attr_if_possible(nested_decoder, "pad_token_id", pad_token_id)
generation_config = getattr(model, "generation_config", None)
if generation_config is not None:
_set_attr_if_possible(generation_config, "decoder_start_token_id", decoder_start_token_id)
_set_attr_if_possible(generation_config, "pad_token_id", pad_token_id)
def write_corrector_recipe(
output_dir: str | Path,
*,
clean_manifest: str | Path | None = None,
pairs_source: str | Path | None = None,
hf_dataset: str | None = None,
hf_split: str = "train",
limit: int | None = None,
seed: int = 13,
) -> Path:
sources = [clean_manifest is not None, pairs_source is not None, hf_dataset is not None]
if sum(1 for item in sources if item) != 1:
raise DataValidationError("train-corrector requires exactly one of --clean-manifest, --pairs-source, or --hf-dataset")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
pairs_path = out / "correction_pairs.jsonl"
if pairs_source is not None:
_copy_pairs(pairs_source, pairs_path, limit=limit)
elif hf_dataset is not None:
prepare_hf_correction_pairs(hf_dataset, pairs_path, split=hf_split, limit=limit)
else:
entries = load_manifest(clean_manifest or "")
pairs = _synthetic_pairs(entries, limit=limit, seed=seed)
with pairs_path.open("w", encoding="utf-8") as handle:
for noisy, clean in pairs:
handle.write(json.dumps({"noisy_text": noisy, "clean_text": clean}, ensure_ascii=False) + "\n")
config_path = out / "corrector_config.json"
config_path.write_text(
json.dumps(
{
"task": "ocr_text_correction",
"pairs": str(pairs_path),
"recommended_model_family": "compact seq2seq or encoder-decoder text corrector",
"metrics": ["cer", "wer", "exact_line_accuracy"],
},
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
(out / "README.md").write_text(
"# Corrector Training Recipe\n\n"
f"Pairs: `{pairs_path}`\n\n"
"Use these noisy-clean pairs with a compact text-to-text model. Keep Nepali and English validation sets separate.\n",
encoding="utf-8",
)
return config_path
def generate_correction_pairs(
manifest_path: str | Path,
output_path: str | Path,
*,
engine_name: str = "tesseract",
model_config: str | Path | None = None,
limit: int | None = None,
only_errors: bool = False,
) -> Path:
from .engines import create_engine
entries = load_manifest(manifest_path)
if not entries:
raise DataValidationError("generate-correction-pairs requires a non-empty manifest")
engine_kwargs: dict[str, Any] = {}
if engine_name in {"candidate", "ours"}:
engine_kwargs["model_config"] = model_config
engine = create_engine(engine_name, **engine_kwargs)
out = Path(output_path)
out.parent.mkdir(parents=True, exist_ok=True)
written = 0
with out.open("w", encoding="utf-8") as handle:
for entry in entries:
if limit is not None and written >= limit:
break
recognized = engine.recognize(Path(entry.image_path))
noisy = normalize_ocr_text("\n".join(line.text for page in recognized.pages for line in page.text_lines))
clean = normalize_ocr_text(entry.text)
if only_errors and noisy == clean:
continue
handle.write(
json.dumps(
{
"sample_id": entry.sample_id,
"engine": engine_name,
"noisy_text": noisy,
"clean_text": clean,
"slices": list(entry.metadata.get("slices") or []) if entry.metadata else [],
},
ensure_ascii=False,
)
+ "\n"
)
written += 1
if written == 0:
raise DataValidationError("No correction pairs generated")
return out
def write_hf_text_corrector_recipe(
pairs_source: str | Path,
output_dir: str | Path,
*,
base_model: str = "google/byt5-small",
max_source_length: int = 256,
max_target_length: int = 256,
per_device_train_batch_size: int = 16,
per_device_eval_batch_size: int = 16,
learning_rate: float = 5e-5,
num_train_epochs: float = 3.0,
warmup_steps: int = 0,
logging_steps: int = 10,
save_total_limit: int = 2,
eval_ratio: float = 0.1,
seed: int = 13,
run: bool = False,
) -> Path:
pairs = _load_correction_pairs(Path(pairs_source))
if len(pairs) < 2:
raise DataValidationError("hf text corrector training requires at least two correction pairs")
if eval_ratio <= 0 or eval_ratio >= 1:
raise DataValidationError("hf text corrector eval_ratio must be between 0 and 1")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
rng = random.Random(seed)
shuffled = list(pairs)
rng.shuffle(shuffled)
eval_count = min(max(1, round(len(shuffled) * eval_ratio)), len(shuffled) - 1)
eval_pairs = shuffled[:eval_count]
train_pairs = shuffled[eval_count:]
train_pairs_path = out / "train_pairs.jsonl"
eval_pairs_path = out / "eval_pairs.jsonl"
_write_correction_pairs_jsonl(train_pairs, train_pairs_path)
_write_correction_pairs_jsonl(eval_pairs, eval_pairs_path)
config = HfTextCorrectorTrainingConfig(
train_pairs=str(train_pairs_path),
eval_pairs=str(eval_pairs_path),
output_dir=str(out),
base_model=base_model,
max_source_length=max_source_length,
max_target_length=max_target_length,
per_device_train_batch_size=per_device_train_batch_size,
per_device_eval_batch_size=per_device_eval_batch_size,
learning_rate=learning_rate,
num_train_epochs=num_train_epochs,
warmup_steps=warmup_steps,
logging_steps=logging_steps,
save_total_limit=save_total_limit,
predict_with_generate=True,
generation_max_length=max_target_length,
)
config_path = out / "hf_text_corrector_config.json"
config_path.write_text(json.dumps(config.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(out / "README.md").write_text(
"\n".join(
[
"# Hugging Face Text Corrector Training Recipe",
"",
f"Base model: `{base_model}`",
f"Pairs source: `{pairs_source}` ({len(pairs)} pairs)",
f"Train pairs: `{train_pairs_path}` ({len(train_pairs)} pairs)",
f"Eval pairs: `{eval_pairs_path}` ({len(eval_pairs)} pairs)",
"",
"Install the backend dependencies first:",
"",
"```bash",
"python -m pip install -e '.[hf-recognizer]'",
"```",
"",
"Run in-process:",
"",
"```bash",
f"ocrtech train-corrector --backend hf-seq2seq --pairs-source {pairs_source} --out {out} --run",
"```",
]
)
+ "\n",
encoding="utf-8",
)
if run:
run_hf_text_corrector_training(config_path)
return config_path
def run_hf_text_corrector_training(config_path: str | Path) -> Path:
config = HfTextCorrectorTrainingConfig.from_path(config_path)
train_pairs = _load_correction_pairs(Path(config.train_pairs))
eval_pairs = _load_correction_pairs(Path(config.eval_pairs))
if not train_pairs:
raise DataValidationError("hf text corrector training requires non-empty train_pairs")
if not eval_pairs:
raise DataValidationError("hf text corrector training requires non-empty eval_pairs")
try:
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, DataCollatorForSeq2Seq, Seq2SeqTrainer, Seq2SeqTrainingArguments
except ImportError as exc:
raise DataValidationError("hf text corrector training requires transformers. Install ocr-tech[hf-recognizer].") from exc
output_dir = Path(config.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
tokenizer = AutoTokenizer.from_pretrained(config.base_model, use_fast=False)
model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model)
model.generation_config.max_length = config.generation_max_length
train_dataset = _TextCorrectionDataset(train_pairs, tokenizer, config.max_source_length, config.max_target_length)
eval_dataset = _TextCorrectionDataset(eval_pairs, tokenizer, config.max_source_length, config.max_target_length)
training_args = Seq2SeqTrainingArguments(
output_dir=str(output_dir / "checkpoints"),
eval_strategy="epoch",
save_strategy="epoch",
learning_rate=config.learning_rate,
per_device_train_batch_size=config.per_device_train_batch_size,
per_device_eval_batch_size=config.per_device_eval_batch_size,
num_train_epochs=config.num_train_epochs,
warmup_steps=config.warmup_steps,
logging_steps=config.logging_steps,
save_total_limit=config.save_total_limit,
predict_with_generate=config.predict_with_generate,
generation_max_length=config.generation_max_length,
remove_unused_columns=False,
report_to=[],
)
trainer = Seq2SeqTrainer(
model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
processing_class=tokenizer,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model),
compute_metrics=_hf_text_corrector_metrics_factory(tokenizer),
)
trainer.train()
final_model_dir = output_dir / "final-model"
trainer.save_model(str(final_model_dir))
tokenizer.save_pretrained(str(final_model_dir))
metrics = trainer.evaluate(max_length=config.generation_max_length)
metrics_path = output_dir / "training_metrics.json"
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return final_model_dir
def write_recognizer_export_recipe(
training_config: str | Path,
checkpoint: str | Path,
output_dir: str | Path,
*,
inference_dir: str | Path | None = None,
paddleocr_dir: str | Path = "PaddleOCR",
run: bool = False,
) -> Path:
config_path = Path(training_config)
checkpoint_path = _paddle_checkpoint_prefix(Path(checkpoint))
if not config_path.exists():
raise DataValidationError(f"training_config does not exist: {config_path}")
if run and not _paddle_checkpoint_exists(checkpoint_path):
raise DataValidationError(f"checkpoint does not exist: {checkpoint_path}")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
save_dir = Path(inference_dir) if inference_dir else out / "inference"
export_spec = {
"training_config": str(config_path),
"checkpoint": str(checkpoint_path),
"paddleocr_dir": str(paddleocr_dir),
"save_inference_dir": str(save_dir),
"command": [
"python",
str(Path(paddleocr_dir) / "tools" / "export_model.py"),
"-c",
str(config_path),
"-o",
f"Global.pretrained_model={checkpoint_path}",
f"Global.save_inference_dir={save_dir}",
],
}
export_path = out / "export-recognizer.json"
export_path.write_text(json.dumps(export_spec, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(out / "EXPORT.md").write_text(
"# Recognizer Export\n\n"
"Run this after training has produced a PaddleOCR checkpoint.\n\n"
"```bash\n"
+ " ".join(export_spec["command"])
+ "\n```\n",
encoding="utf-8",
)
if run:
completed = subprocess.run(export_spec["command"], check=False)
if completed.returncode != 0:
raise DataValidationError(f"PaddleOCR export failed with exit code {completed.returncode}")
return export_path
def _paddle_checkpoint_prefix(checkpoint: Path) -> Path:
if checkpoint.name.endswith(".pdparams"):
return checkpoint.with_name(checkpoint.name[: -len(".pdparams")])
return checkpoint
def _paddle_checkpoint_exists(checkpoint_prefix: Path) -> bool:
return checkpoint_prefix.exists() or checkpoint_prefix.with_name(f"{checkpoint_prefix.name}.pdparams").is_file()
def package_paddleocr_model(
inference_dir: str | Path,
output_dir: str | Path,
*,
model_id: str,
dictionary_path: str | Path,
base_model: str = PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
train_manifest: str | Path | None = None,
eval_manifest: str | Path | None = None,
training_summary: str | Path | None = None,
metrics_report: str | Path | None = None,
admission_validation_report: str | Path | None = None,
) -> Path:
source_dir = Path(inference_dir)
dictionary = Path(dictionary_path)
if not source_dir.exists() or not source_dir.is_dir():
raise DataValidationError(f"inference_dir does not exist or is not a directory: {source_dir}")
if not dictionary.exists() or not dictionary.is_file():
raise DataValidationError(f"dictionary_path does not exist or is not a file: {dictionary}")
model_files = _find_paddle_inference_files(source_dir)
out = Path(output_dir)
artifacts_dir = out / "artifacts"
artifacts_dir.mkdir(parents=True, exist_ok=True)
copied: list[Path] = []
for file_path in model_files:
target = artifacts_dir / file_path.name
shutil.copy2(file_path, target)
copied.append(target)
dict_target = artifacts_dir / dictionary.name
shutil.copy2(dictionary, dict_target)
copied.append(dict_target)
card_path = out / "model-card.json"
write_model_card(
card_path,
model_id=model_id,
backend="paddleocr",
base_model=base_model,
artifact_paths=[path.relative_to(out) for path in copied],
backend_kwargs={
"lang": "ne",
"text_recognition_model_name": "devanagari_PP-OCRv3_mobile_rec",
"text_recognition_model_dir": str(artifacts_dir.relative_to(out)),
},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
metrics_report=metrics_report,
admission_validation_report=admission_validation_report,
artifact_base_path=out,
)
(out / "PACKAGE.md").write_text(
"# Packaged PaddleOCR Candidate\n\n"
f"Model card: `{card_path}`\n"
f"Artifacts: `{artifacts_dir}`\n\n"
"Audit before benchmarking:\n\n"
"```bash\n"
f"ocrtech audit-model {card_path}\n"
"```\n",
encoding="utf-8",
)
return card_path
def package_hf_recognizer_model(
model_dir: str | Path,
output_dir: str | Path,
*,
model_id: str,
base_model: str,
train_manifest: str | Path | None = None,
eval_manifest: str | Path | None = None,
training_summary: str | Path | None = None,
metrics_report: str | Path | None = None,
admission_validation_report: str | Path | None = None,
max_new_tokens: int = 128,
) -> Path:
source_dir = Path(model_dir)
if not source_dir.exists() or not source_dir.is_dir():
raise DataValidationError(f"model_dir does not exist or is not a directory: {source_dir}")
out = Path(output_dir)
artifacts_dir = out / "artifacts" / "model"
if artifacts_dir.exists():
shutil.rmtree(artifacts_dir)
artifacts_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(source_dir, artifacts_dir)
card_path = out / "model-card.json"
write_model_card(
card_path,
model_id=model_id,
backend="hf_vision_encoder_decoder",
base_model=base_model,
artifact_paths=[artifacts_dir.relative_to(out)],
backend_kwargs={
"model_dir": str(artifacts_dir.relative_to(out)),
"processor_dir": str(artifacts_dir.relative_to(out)),
"max_new_tokens": max_new_tokens,
},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
metrics_report=metrics_report,
admission_validation_report=admission_validation_report,
artifact_base_path=out,
)
(out / "PACKAGE.md").write_text(
"# Packaged Hugging Face OCR Candidate\n\n"
f"Model card: `{card_path}`\n"
f"Artifacts: `{artifacts_dir}`\n\n"
"Audit before benchmarking:\n\n"
"```bash\n"
f"ocrtech audit-model {card_path}\n"
"```\n",
encoding="utf-8",
)
return card_path
def finalize_hf_recognizer_run(
run_dir: str | Path,
*,
output_dir: str | Path | None = None,
model_id: str,
base_model: str,
train_manifest: str | Path | None = None,
eval_manifest: str | Path | None = None,
training_summary: str | Path | None = None,
admission_validation_report: str | Path | None = None,
max_new_tokens: int = 128,
) -> HfRecognizerFinalization:
run_path = Path(run_dir)
if not run_path.exists() or not run_path.is_dir():
raise DataValidationError(f"run_dir does not exist or is not a directory: {run_path}")
final_model_dir = run_path / "final-model"
metrics_path = run_path / "training_metrics.json"
if not final_model_dir.exists() or not final_model_dir.is_dir():
raise DataValidationError(f"HF recognizer run is not finished; missing final model directory: {final_model_dir}")
if not metrics_path.exists() or not metrics_path.is_file():
raise DataValidationError(f"HF recognizer run is not finished; missing training metrics report: {metrics_path}")
try:
metrics_payload = json.loads(metrics_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise DataValidationError(f"Invalid training metrics JSON {metrics_path}: {exc}") from exc
if not isinstance(metrics_payload, dict):
raise DataValidationError(f"training metrics report must be a JSON object: {metrics_path}")
out = Path(output_dir) if output_dir is not None else run_path / "finalized"
package_dir = out / "package"
audit_dir = out / "audit"
report_json = out / "hf-recognizer-finalization.json"
report_md = out / "hf-recognizer-finalization.md"
out.mkdir(parents=True, exist_ok=True)
card_path = package_hf_recognizer_model(
final_model_dir,
package_dir,
model_id=model_id,
base_model=base_model,
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
metrics_report=metrics_path,
admission_validation_report=admission_validation_report,
max_new_tokens=max_new_tokens,
)
audit = audit_model_card(card_path, audit_dir)
result = HfRecognizerFinalization(
run_dir=str(run_path),
package_dir=str(package_dir),
model_card_path=str(card_path),
audit_path=str(audit_dir / "model-audit.json"),
report_json_path=str(report_json),
report_md_path=str(report_md),
audit_passed=audit.passed,
)
report_json.write_text(json.dumps(result.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
report_md.write_text(_render_hf_recognizer_finalization(result), encoding="utf-8")
return result
def _render_hf_recognizer_finalization(result: HfRecognizerFinalization) -> str:
return "\n".join(
[
"# HF Recognizer Finalization",
"",
f"Run directory: `{result.run_dir}`",
f"Package directory: `{result.package_dir}`",
f"Model card: `{result.model_card_path}`",
f"Model audit: `{result.audit_path}`",
f"Audit passed: `{'yes' if result.audit_passed else 'no'}`",
"",
"## Next Claim-Bearing Steps",
"",
"1. Run `ocrtech audit-model` on the packaged model card if the generated audit did not pass.",
"2. Benchmark the packaged candidate against named baselines on the held-out manifest.",
"3. Summarize paired metrics and run `ocrtech validate-claim` before writing any SOTA claim.",
"",
"Example experiment command:",
"",
"```bash",
"ocrtech run-experiment \\",
" --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \\",
" --out outputs/experiment-hf-recognizer-mixed-v2-full \\",
" --baselines candidate,tesseract,surya,stock-paddle,glm-ocr,paddleocr-vl \\",
" --candidate-model-config " + result.model_card_path + " \\",
" --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl",
"```",
"",
]
)
def package_hf_text_corrector_model(
model_dir: str | Path,
output_dir: str | Path,
*,
model_id: str,
base_model: str,
base_engine: str = "tesseract",
base_engine_kwargs: dict[str, Any] | None = None,
train_manifest: str | Path | None = None,
eval_manifest: str | Path | None = None,
metrics_report: str | Path | None = None,
max_new_tokens: int = 256,
) -> Path:
source_dir = Path(model_dir)
if not source_dir.exists() or not source_dir.is_dir():
raise DataValidationError(f"model_dir does not exist or is not a directory: {source_dir}")
out = Path(output_dir)
artifacts_dir = out / "artifacts" / "model"
if artifacts_dir.exists():
shutil.rmtree(artifacts_dir)
artifacts_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(source_dir, artifacts_dir)
card_path = out / "model-card.json"
write_model_card(
card_path,
model_id=model_id,
backend="text_correction_composite",
base_model=base_model,
artifact_paths=[artifacts_dir.relative_to(out)],
backend_kwargs={
"base_engine": base_engine,
"base_engine_kwargs": base_engine_kwargs or {},
"model_dir": str(artifacts_dir.relative_to(out)),
"tokenizer_dir": str(artifacts_dir.relative_to(out)),
"max_new_tokens": max_new_tokens,
},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
metrics_report=metrics_report,
)
(out / "PACKAGE.md").write_text(
"# Packaged Text Correction Composite Candidate\n\n"
f"Model card: `{card_path}`\n"
f"Artifacts: `{artifacts_dir}`\n\n"
"Audit before benchmarking:\n\n"
"```bash\n"
f"ocrtech audit-model {card_path}\n"
"```\n",
encoding="utf-8",
)
return card_path
def _copy_pairs(source: str | Path, target: Path, *, limit: int | None) -> None:
source_path = Path(source)
count = 0
with source_path.open("r", encoding="utf-8") as src, target.open("w", encoding="utf-8") as dst:
for line_number, line in enumerate(src, start=1):
if limit is not None and count >= limit:
break
if not line.strip():
continue
data = json.loads(line)
if "noisy_text" not in data or "clean_text" not in data:
raise DataValidationError(f"Correction pair at {source_path}:{line_number} needs noisy_text and clean_text")
dst.write(json.dumps({"noisy_text": data["noisy_text"], "clean_text": data["clean_text"]}, ensure_ascii=False) + "\n")
count += 1
def _synthetic_pairs(entries: list[ManifestEntry], *, limit: int | None, seed: int) -> list[tuple[str, str]]:
rng = random.Random(seed)
pairs: list[tuple[str, str]] = []
for entry in entries:
clean = normalize_ocr_text(entry.text)
noisy = _corrupt_text(clean, rng)
if noisy != clean:
pairs.append((noisy, clean))
if limit is not None and len(pairs) >= limit:
break
if not pairs:
raise DataValidationError("No correction pairs generated")
return pairs
def _load_correction_pairs(path: Path) -> list[dict[str, str]]:
pairs: list[dict[str, str]] = []
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
if not line.strip():
continue
data = json.loads(line)
noisy = data.get("noisy_text")
clean = data.get("clean_text")
if not isinstance(noisy, str) or not isinstance(clean, str):
raise DataValidationError(f"Correction pair at {path}:{line_number} needs string noisy_text and clean_text")
pairs.append({"noisy_text": normalize_ocr_text(noisy), "clean_text": normalize_ocr_text(clean)})
if not pairs:
raise DataValidationError(f"No correction pairs found in {path}")
return pairs
def _write_correction_pairs_jsonl(pairs: list[dict[str, str]], path: Path) -> None:
with path.open("w", encoding="utf-8") as handle:
for pair in pairs:
handle.write(json.dumps(pair, ensure_ascii=False) + "\n")
def _corrupt_text(text: str, rng: random.Random) -> str:
chars = list(text)
if len(chars) < 2:
return text
operations = max(1, len(chars) // 24)
for _ in range(operations):
index = rng.randrange(len(chars))
draw = rng.random()
if draw < 0.4:
chars.pop(index)
if not chars:
break
elif draw < 0.7:
chars[index] = " "
else:
chars.insert(index, chars[index])
return "".join(chars)
def _find_paddle_inference_files(source_dir: Path) -> list[Path]:
files = [path for path in source_dir.iterdir() if path.is_file()]
if not files:
raise DataValidationError(f"inference_dir contains no files: {source_dir}")
parameter_files = sorted(path for path in files if path.suffix == ".pdiparams" or path.name.endswith(".pdiparams"))
model_files = sorted(path for path in files if path.suffix in {".pdmodel", ".json"} or path.name in {"inference.pdmodel", "inference.json"})
config_files = sorted(path for path in files if path.name in {"inference.yml", "inference.yaml"})
info_files = sorted(path for path in files if path.name.endswith(".pdiparams.info"))
if not parameter_files:
raise DataValidationError(f"inference_dir is missing PaddleOCR parameter file (*.pdiparams): {source_dir}")
if not model_files:
raise DataValidationError(f"inference_dir is missing PaddleOCR model file (*.pdmodel or *.json): {source_dir}")
return [*model_files, *parameter_files, *config_files, *info_files]
class _VisionTextDataset:
def __init__(self, entries: list[ManifestEntry], processor: Any, image_module: Any, max_target_length: int) -> None:
self.entries = entries
self.processor = processor
self.image_module = image_module
self.max_target_length = max_target_length
def __len__(self) -> int:
return len(self.entries)
def __getitem__(self, index: int) -> dict[str, Any]:
entry = self.entries[index]
image = self.image_module.open(Path(entry.image_path)).convert("RGB")
pixel_values = self.processor(image, return_tensors="pt").pixel_values.squeeze(0)
labels = self.processor.tokenizer(
entry.text,
padding=False,
truncation=True,
max_length=self.max_target_length,
return_tensors="pt",
).input_ids.squeeze(0)
return {"pixel_values": pixel_values, "labels": labels}
class _VisionTextCollator:
def __init__(self, processor: Any, torch: Any) -> None:
self.processor = processor
self.torch = torch
def __call__(self, features: list[dict[str, Any]]) -> dict[str, Any]:
pixel_values = self.torch.stack([feature["pixel_values"] for feature in features])
label_features = [{"input_ids": feature["labels"]} for feature in features]
labels = self.processor.tokenizer.pad(label_features, padding=True, return_tensors="pt").input_ids
labels = labels.masked_fill(labels == self.processor.tokenizer.pad_token_id, -100)
return {"pixel_values": pixel_values, "labels": labels}
class _TextCorrectionDataset:
def __init__(self, pairs: list[dict[str, str]], tokenizer: Any, max_source_length: int, max_target_length: int) -> None:
self.pairs = pairs
self.tokenizer = tokenizer
self.max_source_length = max_source_length
self.max_target_length = max_target_length
def __len__(self) -> int:
return len(self.pairs)
def __getitem__(self, index: int) -> dict[str, Any]:
pair = self.pairs[index]
model_inputs = self.tokenizer(
pair["noisy_text"],
padding=False,
truncation=True,
max_length=self.max_source_length,
)
labels = self.tokenizer(
text_target=pair["clean_text"],
padding=False,
truncation=True,
max_length=self.max_target_length,
)
model_inputs["labels"] = labels["input_ids"]
return model_inputs
def _hf_compute_metrics_factory(processor: Any):
def compute_metrics(eval_pred: Any) -> dict[str, float]:
prediction_ids = eval_pred.predictions
label_ids = eval_pred.label_ids
if isinstance(prediction_ids, tuple):
prediction_ids = prediction_ids[0]
predicted_text = processor.batch_decode(
_sanitize_batch_token_ids(processor.tokenizer, prediction_ids),
skip_special_tokens=True,
)
reference_text = processor.batch_decode(
_sanitize_batch_token_ids(processor.tokenizer, label_ids),
skip_special_tokens=True,
)
return {
"cer": sum(cer(pred, ref) for pred, ref in zip(predicted_text, reference_text, strict=True)) / len(reference_text),
"wer": sum(wer(pred, ref) for pred, ref in zip(predicted_text, reference_text, strict=True)) / len(reference_text),
"exact_line_accuracy": exact_line_accuracy(predicted_text, reference_text),
}
return compute_metrics
def _hf_text_corrector_metrics_factory(tokenizer: Any):
def compute_metrics(eval_pred: Any) -> dict[str, float]:
prediction_ids = eval_pred.predictions
label_ids = eval_pred.label_ids
if isinstance(prediction_ids, tuple):
prediction_ids = prediction_ids[0]
predicted_text = [
normalize_ocr_text(text)
for text in tokenizer.batch_decode(_sanitize_batch_token_ids(tokenizer, prediction_ids), skip_special_tokens=True)
]
reference_text = [
normalize_ocr_text(text)
for text in tokenizer.batch_decode(_sanitize_batch_token_ids(tokenizer, label_ids), skip_special_tokens=True)
]
return {
"cer": sum(cer(pred, ref) for pred, ref in zip(predicted_text, reference_text, strict=True)) / len(reference_text),
"wer": sum(wer(pred, ref) for pred, ref in zip(predicted_text, reference_text, strict=True)) / len(reference_text),
"exact_line_accuracy": exact_line_accuracy(predicted_text, reference_text),
}
return compute_metrics
def _sanitize_batch_token_ids(tokenizer: Any, token_rows: Any) -> list[list[int]]:
rows = token_rows.tolist() if hasattr(token_rows, "tolist") else token_rows
pad_token_id = _fallback_token_id(tokenizer)
vocab_size = _tokenizer_vocab_size(tokenizer)
sanitized_rows: list[list[int]] = []
for row in rows:
values = row.tolist() if hasattr(row, "tolist") else row
sanitized_rows.append([_sanitize_token_id(token, pad_token_id=pad_token_id, vocab_size=vocab_size) for token in values])
return sanitized_rows
def _sanitize_token_id(token: Any, *, pad_token_id: int, vocab_size: int | None) -> int:
try:
value = int(token)
except (TypeError, ValueError, OverflowError):
return pad_token_id
if value == -100:
return pad_token_id
if value < 0:
return pad_token_id
if vocab_size is not None and value >= vocab_size:
return pad_token_id
return value
def _tokenizer_vocab_size(tokenizer: Any) -> int | None:
vocab_size = getattr(tokenizer, "vocab_size", None)
if vocab_size is not None:
return int(vocab_size)
try:
return len(tokenizer)
except TypeError:
return None
def _fallback_token_id(tokenizer: Any) -> int:
for attribute in ("pad_token_id", "eos_token_id", "bos_token_id"):
value = getattr(tokenizer, attribute, None)
if value is not None:
return int(value)
return 0
"""Training recipe generation for recognizer and corrector models."""
from __future__ import annotations
import json
import math
import random
import shlex
import shutil
import subprocess
import tarfile
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .admission import assert_recognizer_family_not_stopped
from .datasets import audit_recognizer_corpus
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, prepare_hf_correction_pairs, sha256_file, write_manifest
from .metrics import cer, exact_line_accuracy, wer
from .models import audit_model_card, write_model_card
from .normalization import dictionary_from_texts, normalize_ocr_text
PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED = (
"https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/"
"devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams"
)
def write_recognizer_recipe(
train_manifest: str | Path,
eval_manifest: str | Path,
output_dir: str | Path,
*,
base_model: str = PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
dictionary_path: str | Path | None = None,
train_batch_size: int = 64,
eval_batch_size: int = 64,
learning_rate: float = 0.001,
epochs: float | None = None,
failure_review: str | Path | None = None,
run: bool = False,
) -> Path:
return write_paddle_recognizer_recipe(
train_manifest,
eval_manifest,
output_dir,
base_model=base_model,
dictionary_path=dictionary_path,
train_batch_size=train_batch_size,
eval_batch_size=eval_batch_size,
learning_rate=learning_rate,
epochs=epochs,
failure_review=failure_review,
run=run,
)
def write_paddle_recognizer_recipe(
train_manifest: str | Path,
eval_manifest: str | Path,
output_dir: str | Path,
*,
base_model: str = PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
dictionary_path: str | Path | None = None,
train_batch_size: int = 64,
eval_batch_size: int = 64,
learning_rate: float = 0.001,
epochs: float | None = None,
failure_review: str | Path | None = None,
run: bool = False,
) -> Path:
if train_batch_size <= 0:
raise DataValidationError("train_batch_size must be positive")
if eval_batch_size <= 0:
raise DataValidationError("eval_batch_size must be positive")
if learning_rate <= 0:
raise DataValidationError("learning_rate must be positive")
if epochs is not None and epochs <= 0:
raise DataValidationError("epochs must be positive when provided")
if failure_review is not None:
assert_recognizer_family_not_stopped(failure_review, backend="paddleocr", base_model=base_model)
train_entries = load_manifest(train_manifest)
eval_entries = load_manifest(eval_manifest)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
labels_dir = out / "labels"
labels_dir.mkdir(exist_ok=True)
train_label = labels_dir / "train.txt"
eval_label = labels_dir / "eval.txt"
_write_paddle_label_file(train_entries, train_label)
_write_paddle_label_file(eval_entries, eval_label)
dict_path = Path(dictionary_path) if dictionary_path else out / "nep_eng_dict.txt"
if dictionary_path is None:
chars = dictionary_from_texts([entry.text for entry in [*train_entries, *eval_entries]])
dict_path.write_text("\n".join(chars) + "\n", encoding="utf-8")
elif not dict_path.exists():