Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/mps_recognizer.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 24.3 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/mps_recognizer.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/mps_recognizer.py
-
curl -L -o mps_recognizer.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/mps_recognizer.py
24.3 kB
| """PyTorch PP-OCRv5 CTC recognizer training with genuine Apple MPS execution. | |
| This backend vendors the Apache-2.0 PaddleOCR2Pytorch PPLCNetV3 and SVTR | |
| modules at a pinned revision. It intentionally trains only the CTC branch of | |
| PP-OCRv5 MultiHead: the upstream PyTorch NRTR path is not training complete. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import random | |
| import string | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from typing import Any, Sequence | |
| import numpy as np | |
| from PIL import Image | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from torch.utils.data import DataLoader, Dataset | |
| from ._vendor.paddleocr2pytorch.modeling.backbones.rec_lcnetv3 import PPLCNetV3 | |
| from ._vendor.paddleocr2pytorch.modeling.heads.rec_ctc_head import CTCHead | |
| from ._vendor.paddleocr2pytorch.modeling.necks.rnn import SequenceEncoder | |
| from .errors import DataValidationError | |
| from .manifest import ManifestEntry, load_manifest | |
| from .normalization import normalize_ocr_text | |
| PADDLEOCR2PYTORCH_REVISION = "c05307fbd61e575d55f0fef0334026c7536aec76" | |
| MPS_RECOGNIZER_BACKEND = "pytorch_ppocrv5_ctc" | |
| CTC_BLANK_INDEX = 0 | |
| CTC_DOWNSAMPLE_FACTOR = 4 | |
| DEFAULT_WIDTH_BUCKETS = (320, 640, 1024) | |
| class MpsRecognizerTrainingConfig: | |
| train_manifest: str | |
| eval_manifest: str | |
| output_dir: str | |
| dictionary_path: str | |
| device: str = "mps" | |
| require_device: bool = True | |
| image_height: int = 48 | |
| width_buckets: tuple[int, ...] = DEFAULT_WIDTH_BUCKETS | |
| max_target_length: int = 128 | |
| train_batch_size: int = 8 | |
| eval_batch_size: int = 8 | |
| learning_rate: float = 5e-4 | |
| epochs: float = 3.0 | |
| num_workers: int = 0 | |
| seed: int = 13 | |
| max_steps: int | None = None | |
| initial_checkpoint: str | None = None | |
| schema_version: int = 1 | |
| backend: str = MPS_RECOGNIZER_BACKEND | |
| source_revision: str = PADDLEOCR2PYTORCH_REVISION | |
| def to_dict(self) -> dict[str, Any]: | |
| payload = asdict(self) | |
| payload["width_buckets"] = list(self.width_buckets) | |
| return payload | |
| def from_path(cls, path: str | Path) -> "MpsRecognizerTrainingConfig": | |
| payload = json.loads(Path(path).read_text(encoding="utf-8")) | |
| payload["width_buckets"] = tuple(int(value) for value in payload["width_buckets"]) | |
| return cls(**payload) | |
| def normalized_ctc_label(text: str) -> str: | |
| """Match the single-line normalization used by Paddle recognizer labels.""" | |
| return " ".join(normalize_ocr_text(text).replace("\t", " ").split()) | |
| def ctc_required_timesteps(text: str) -> int: | |
| """Return exact CTC demand: symbols plus blanks needed between repeats.""" | |
| label = normalized_ctc_label(text) | |
| repeats = sum(left == right for left, right in zip(label, label[1:])) | |
| return len(label) + repeats | |
| def select_width_bucket( | |
| required_timesteps: int, | |
| width_buckets: Sequence[int] = DEFAULT_WIDTH_BUCKETS, | |
| *, | |
| minimum_pixels: int = 0, | |
| ) -> int: | |
| buckets = _validate_width_buckets(width_buckets) | |
| required_pixels = max(minimum_pixels, required_timesteps * CTC_DOWNSAMPLE_FACTOR) | |
| for width in buckets: | |
| if width >= required_pixels: | |
| return width | |
| raise DataValidationError( | |
| f"sample requires {required_timesteps} CTC steps ({required_pixels}px), " | |
| f"but largest width bucket is {buckets[-1]}px/{buckets[-1] // CTC_DOWNSAMPLE_FACTOR} steps" | |
| ) | |
| def _validate_width_buckets(width_buckets: Sequence[int]) -> tuple[int, ...]: | |
| buckets = tuple(int(value) for value in width_buckets) | |
| if not buckets or any(value <= 0 for value in buckets): | |
| raise DataValidationError("width_buckets must contain positive integers") | |
| if tuple(sorted(set(buckets))) != buckets: | |
| raise DataValidationError("width_buckets must be strictly increasing") | |
| if any(value % CTC_DOWNSAMPLE_FACTOR for value in buckets): | |
| raise DataValidationError("width_buckets must be divisible by 4") | |
| return buckets | |
| def _validate_config(config: MpsRecognizerTrainingConfig) -> None: | |
| if config.backend != MPS_RECOGNIZER_BACKEND: | |
| raise DataValidationError(f"unsupported MPS recognizer backend: {config.backend}") | |
| if config.image_height != 48: | |
| raise DataValidationError("PPLCNetV3 MPS training currently requires image_height=48") | |
| _validate_width_buckets(config.width_buckets) | |
| if config.max_target_length <= 0: | |
| raise DataValidationError("max_target_length must be positive") | |
| if config.train_batch_size <= 0 or config.eval_batch_size <= 0: | |
| raise DataValidationError("batch sizes must be positive") | |
| if config.learning_rate <= 0 or not math.isfinite(config.learning_rate): | |
| raise DataValidationError("learning_rate must be positive and finite") | |
| if config.epochs <= 0 or not math.isfinite(config.epochs): | |
| raise DataValidationError("epochs must be positive and finite") | |
| if config.num_workers < 0: | |
| raise DataValidationError("num_workers must be non-negative") | |
| if config.max_steps is not None and config.max_steps <= 0: | |
| raise DataValidationError("max_steps must be positive when provided") | |
| def _dictionary_characters(path: str | Path) -> list[str]: | |
| characters: list[str] = [] | |
| seen: set[str] = set() | |
| for line_number, raw in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), start=1): | |
| if not raw or raw.isspace(): | |
| continue | |
| if len(raw) != 1: | |
| raise DataValidationError( | |
| f"CTC dictionary tokens must be one Unicode code point; line {line_number} has {raw!r}" | |
| ) | |
| if raw in seen: | |
| raise DataValidationError(f"duplicate CTC dictionary character at line {line_number}: {raw!r}") | |
| seen.add(raw) | |
| characters.append(raw) | |
| if not characters: | |
| raise DataValidationError("CTC dictionary must not be empty") | |
| if " " in seen: | |
| characters.remove(" ") | |
| characters.append(" ") | |
| return characters | |
| def _generated_dictionary(entries: Sequence[ManifestEntry]) -> list[str]: | |
| characters = {char for char in string.printable if not char.isspace()} | |
| for entry in entries: | |
| characters.update(char for char in normalized_ctc_label(entry.text) if not char.isspace()) | |
| return sorted(characters) | |
| def _audit_entries( | |
| entries: Sequence[ManifestEntry], | |
| characters: Sequence[str], | |
| config: MpsRecognizerTrainingConfig, | |
| *, | |
| split_name: str, | |
| ) -> dict[str, Any]: | |
| if not entries: | |
| raise DataValidationError(f"{split_name} manifest must not be empty") | |
| allowed = set(characters) | |
| bucket_counts = {str(width): 0 for width in config.width_buckets} | |
| max_label_length = 0 | |
| max_required = 0 | |
| wider_than_320 = 0 | |
| for entry in entries: | |
| label = normalized_ctc_label(entry.text) | |
| if not label: | |
| raise DataValidationError(f"{split_name} sample {entry.sample_id} has an empty normalized label") | |
| if len(label) > config.max_target_length: | |
| raise DataValidationError( | |
| f"{split_name} sample {entry.sample_id} has {len(label)} characters; " | |
| f"max_target_length={config.max_target_length}" | |
| ) | |
| missing = sorted({char for char in label if char not in allowed}) | |
| if missing: | |
| raise DataValidationError( | |
| f"{split_name} sample {entry.sample_id} has characters absent from dictionary: {missing!r}" | |
| ) | |
| required = ctc_required_timesteps(label) | |
| width = select_width_bucket(required, config.width_buckets) | |
| bucket_counts[str(width)] += 1 | |
| max_label_length = max(max_label_length, len(label)) | |
| max_required = max(max_required, required) | |
| wider_than_320 += int(required > 320 // CTC_DOWNSAMPLE_FACTOR) | |
| return { | |
| "sample_count": len(entries), | |
| "max_label_length": max_label_length, | |
| "max_ctc_required_timesteps": max_required, | |
| "samples_requiring_more_than_80_steps": wider_than_320, | |
| "minimum_label_bucket_counts": bucket_counts, | |
| } | |
| def write_mps_recognizer_recipe( | |
| train_manifest: str | Path, | |
| eval_manifest: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| dictionary_path: str | Path | None = None, | |
| device: str = "mps", | |
| require_device: bool = True, | |
| width_buckets: Sequence[int] = DEFAULT_WIDTH_BUCKETS, | |
| max_target_length: int = 128, | |
| train_batch_size: int = 8, | |
| eval_batch_size: int = 8, | |
| learning_rate: float = 5e-4, | |
| epochs: float = 3.0, | |
| num_workers: int = 0, | |
| seed: int = 13, | |
| max_steps: int | None = None, | |
| initial_checkpoint: str | Path | None = None, | |
| run: bool = False, | |
| ) -> Path: | |
| """Write a validated recipe and optionally execute the PyTorch trainer.""" | |
| train_path = Path(train_manifest).resolve() | |
| eval_path = Path(eval_manifest).resolve() | |
| output = Path(output_dir).resolve() | |
| output.mkdir(parents=True, exist_ok=True) | |
| train_entries = load_manifest(train_path) | |
| eval_entries = load_manifest(eval_path) | |
| if dictionary_path is None: | |
| dictionary_chars = _generated_dictionary([*train_entries, *eval_entries]) | |
| else: | |
| dictionary_chars = [char for char in _dictionary_characters(dictionary_path) if char != " "] | |
| recipe_dictionary = output / "paddle_mps_dict.txt" | |
| recipe_dictionary.write_text("".join(f"{char}\n" for char in dictionary_chars), encoding="utf-8") | |
| config = MpsRecognizerTrainingConfig( | |
| train_manifest=str(train_path), | |
| eval_manifest=str(eval_path), | |
| output_dir=str(output), | |
| dictionary_path=str(recipe_dictionary), | |
| device=device, | |
| require_device=require_device, | |
| width_buckets=_validate_width_buckets(width_buckets), | |
| max_target_length=max_target_length, | |
| train_batch_size=train_batch_size, | |
| eval_batch_size=eval_batch_size, | |
| learning_rate=learning_rate, | |
| epochs=epochs, | |
| num_workers=num_workers, | |
| seed=seed, | |
| max_steps=max_steps, | |
| initial_checkpoint=str(Path(initial_checkpoint).resolve()) if initial_checkpoint else None, | |
| ) | |
| _validate_config(config) | |
| characters = _dictionary_characters(recipe_dictionary) | |
| audit = { | |
| "backend": config.backend, | |
| "source_revision": config.source_revision, | |
| "ctc_downsample_factor": CTC_DOWNSAMPLE_FACTOR, | |
| "train": _audit_entries(train_entries, characters, config, split_name="train"), | |
| "eval": _audit_entries(eval_entries, characters, config, split_name="eval"), | |
| } | |
| config_path = output / "paddle_mps_config.json" | |
| config_path.write_text(json.dumps(config.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (output / "paddle_mps_capacity_audit.json").write_text( | |
| json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| if run: | |
| run_mps_recognizer_training(config) | |
| return config_path | |
| def _contiguous_conv_input(_module: nn.Module, args: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]: | |
| """Work around MPS NativeBatchNormBackward requiring a contiguous conv3 input.""" | |
| return (args[0].contiguous(), *args[1:]) | |
| class CtcOnlyMultiHead(nn.Module): | |
| """The PP-OCRv5 MultiHead CTC path, with the incomplete NRTR path omitted.""" | |
| def __init__(self, in_channels: int, out_channels: int) -> None: | |
| super().__init__() | |
| # Paddle uses use_guide=True while jointly training GTC/NRTR. With no GTC | |
| # branch that detach would prevent all CTC gradients from reaching the | |
| # backbone, so this CTC-only adapter intentionally uses False. | |
| self.ctc_encoder = SequenceEncoder( | |
| in_channels, | |
| encoder_type="svtr", | |
| dims=120, | |
| depth=2, | |
| hidden_dims=120, | |
| kernel_size=[1, 3], | |
| use_guide=False, | |
| ) | |
| self.ctc_head = CTCHead(self.ctc_encoder.out_channels, out_channels=out_channels, fc_decay=1e-5) | |
| self.ctc_encoder.encoder.conv3.register_forward_pre_hook(_contiguous_conv_input) | |
| def forward_logits(self, features: torch.Tensor) -> torch.Tensor: | |
| encoded = self.ctc_encoder(features) | |
| return self.ctc_head.fc(encoded) | |
| def forward(self, features: torch.Tensor) -> torch.Tensor: | |
| logits = self.forward_logits(features) | |
| return logits if self.training else F.softmax(logits, dim=2) | |
| class PpOcrV5CtcRecognizer(nn.Module): | |
| """PPLCNetV3 + SVTR + CTC, compatible with the PP-OCRv5 CTC branch layout.""" | |
| def __init__(self, num_classes: int) -> None: | |
| super().__init__() | |
| if num_classes < 2: | |
| raise DataValidationError("num_classes must include CTC blank plus at least one character") | |
| self.backbone = PPLCNetV3(scale=0.95) | |
| self.head = CtcOnlyMultiHead(self.backbone.out_channels, num_classes) | |
| def forward_logits(self, images: torch.Tensor) -> torch.Tensor: | |
| return self.head.forward_logits(self.backbone(images)) | |
| def forward(self, images: torch.Tensor) -> torch.Tensor: | |
| features = self.backbone(images) | |
| return self.head(features) | |
| class MpsRecognizerDataset(Dataset[dict[str, Any]]): | |
| def __init__( | |
| self, | |
| manifest_path: str | Path, | |
| characters: Sequence[str], | |
| *, | |
| image_height: int = 48, | |
| width_buckets: Sequence[int] = DEFAULT_WIDTH_BUCKETS, | |
| max_target_length: int = 128, | |
| ) -> None: | |
| self.manifest_path = Path(manifest_path) | |
| self.entries = load_manifest(self.manifest_path) | |
| self.character_to_index = {char: index + 1 for index, char in enumerate(characters)} | |
| self.image_height = image_height | |
| self.width_buckets = _validate_width_buckets(width_buckets) | |
| self.max_target_length = max_target_length | |
| def __len__(self) -> int: | |
| return len(self.entries) | |
| def __getitem__(self, index: int) -> dict[str, Any]: | |
| entry = self.entries[index] | |
| label = normalized_ctc_label(entry.text) | |
| if not label or len(label) > self.max_target_length: | |
| raise DataValidationError(f"invalid CTC label length for sample {entry.sample_id}: {len(label)}") | |
| try: | |
| target = torch.tensor([self.character_to_index[char] for char in label], dtype=torch.long) | |
| except KeyError as exc: | |
| raise DataValidationError( | |
| f"sample {entry.sample_id} character is absent from dictionary: {exc.args[0]!r}" | |
| ) from exc | |
| required = ctc_required_timesteps(label) | |
| raw_path = Path(entry.image_path) | |
| image_path = raw_path if raw_path.is_absolute() else self.manifest_path.parent / raw_path | |
| with Image.open(image_path) as source: | |
| rgb = source.convert("RGB") | |
| natural_width = max(1, math.ceil(rgb.width * self.image_height / max(1, rgb.height))) | |
| natural_width = min(natural_width, self.width_buckets[-1]) | |
| bucket_width = select_width_bucket( | |
| required, | |
| self.width_buckets, | |
| minimum_pixels=natural_width, | |
| ) | |
| resized = rgb.resize((natural_width, self.image_height), Image.Resampling.BILINEAR) | |
| # Paddle DecodeImage emits BGR. Copy removes the negative stride from channel reversal. | |
| array = np.asarray(resized, dtype=np.float32)[:, :, ::-1].copy() | |
| image = torch.from_numpy(array).permute(2, 0, 1).contiguous() | |
| image = (image / 255.0 - 0.5) / 0.5 | |
| return { | |
| "image": image, | |
| "target": target, | |
| "target_length": len(label), | |
| "input_length": bucket_width // CTC_DOWNSAMPLE_FACTOR, | |
| "bucket_width": bucket_width, | |
| "sample_id": entry.sample_id, | |
| } | |
| def collate_mps_recognizer(batch: Sequence[dict[str, Any]]) -> dict[str, Any]: | |
| if not batch: | |
| raise DataValidationError("cannot collate an empty recognizer batch") | |
| batch_width = max(int(item["bucket_width"]) for item in batch) | |
| height = int(batch[0]["image"].shape[1]) | |
| images = torch.zeros((len(batch), 3, height, batch_width), dtype=torch.float32) | |
| for index, item in enumerate(batch): | |
| width = int(item["image"].shape[2]) | |
| images[index, :, :, :width] = item["image"] | |
| return { | |
| "images": images, | |
| "targets": torch.cat([item["target"] for item in batch]), | |
| "target_lengths": torch.tensor([item["target_length"] for item in batch], dtype=torch.long), | |
| "input_lengths": torch.tensor([item["input_length"] for item in batch], dtype=torch.long), | |
| "sample_ids": [item["sample_id"] for item in batch], | |
| } | |
| def resolve_training_device(requested: str, *, require_device: bool = True) -> torch.device: | |
| requested = requested.lower() | |
| if requested == "auto": | |
| requested = "mps" if torch.backends.mps.is_available() else "cpu" | |
| if requested == "mps": | |
| available = torch.backends.mps.is_built() and torch.backends.mps.is_available() | |
| if not available: | |
| if require_device: | |
| raise DataValidationError("MPS was requested but torch.backends.mps is unavailable") | |
| return torch.device("cpu") | |
| return torch.device("mps") | |
| if requested == "cpu": | |
| return torch.device("cpu") | |
| raise DataValidationError("device must be one of: mps, cpu, auto") | |
| def ctc_training_loss( | |
| logits: torch.Tensor, | |
| targets: torch.Tensor, | |
| input_lengths: torch.Tensor, | |
| target_lengths: torch.Tensor, | |
| ) -> torch.Tensor: | |
| log_probs = logits.log_softmax(dim=2).permute(1, 0, 2).contiguous() | |
| if logits.device.type == "mps": | |
| # PyTorch does not implement CTCLoss on MPS; the transfer remains differentiable. | |
| log_probs = log_probs.cpu() | |
| return F.ctc_loss( | |
| log_probs, | |
| targets.cpu(), | |
| input_lengths.cpu(), | |
| target_lengths.cpu(), | |
| blank=CTC_BLANK_INDEX, | |
| reduction="mean", | |
| zero_infinity=False, | |
| ) | |
| def _load_initial_checkpoint(model: nn.Module, path: str | None) -> dict[str, int]: | |
| if not path: | |
| return {"loaded_tensors": 0, "skipped_tensors": 0} | |
| payload = torch.load(path, map_location="cpu", weights_only=True) | |
| state = payload.get("model_state_dict", payload) if isinstance(payload, dict) else payload | |
| if not isinstance(state, dict): | |
| raise DataValidationError(f"initial checkpoint does not contain a state dict: {path}") | |
| current = model.state_dict() | |
| compatible = { | |
| key: value | |
| for key, value in state.items() | |
| if key in current and hasattr(value, "shape") and current[key].shape == value.shape | |
| } | |
| model.load_state_dict(compatible, strict=False) | |
| return {"loaded_tensors": len(compatible), "skipped_tensors": len(state) - len(compatible)} | |
| def _tree_to_cpu(value: Any) -> Any: | |
| if isinstance(value, torch.Tensor): | |
| return value.detach().cpu() | |
| if isinstance(value, dict): | |
| return {key: _tree_to_cpu(item) for key, item in value.items()} | |
| if isinstance(value, list): | |
| return [_tree_to_cpu(item) for item in value] | |
| if isinstance(value, tuple): | |
| return tuple(_tree_to_cpu(item) for item in value) | |
| return value | |
| def run_mps_recognizer_training( | |
| config_or_path: MpsRecognizerTrainingConfig | str | Path, | |
| ) -> Path: | |
| """Train on MPS/CPU and write a portable PyTorch state-dict checkpoint.""" | |
| config = ( | |
| config_or_path | |
| if isinstance(config_or_path, MpsRecognizerTrainingConfig) | |
| else MpsRecognizerTrainingConfig.from_path(config_or_path) | |
| ) | |
| _validate_config(config) | |
| device = resolve_training_device(config.device, require_device=config.require_device) | |
| random.seed(config.seed) | |
| np.random.seed(config.seed) | |
| torch.manual_seed(config.seed) | |
| characters = _dictionary_characters(config.dictionary_path) | |
| train_entries = load_manifest(config.train_manifest) | |
| eval_entries = load_manifest(config.eval_manifest) | |
| _audit_entries(train_entries, characters, config, split_name="train") | |
| _audit_entries(eval_entries, characters, config, split_name="eval") | |
| dataset = MpsRecognizerDataset( | |
| config.train_manifest, | |
| characters, | |
| image_height=config.image_height, | |
| width_buckets=config.width_buckets, | |
| max_target_length=config.max_target_length, | |
| ) | |
| loader = DataLoader( | |
| dataset, | |
| batch_size=config.train_batch_size, | |
| shuffle=True, | |
| num_workers=config.num_workers, | |
| pin_memory=False, | |
| collate_fn=collate_mps_recognizer, | |
| drop_last=False, | |
| ) | |
| if not len(loader): | |
| raise DataValidationError("training loader produced no batches") | |
| model = PpOcrV5CtcRecognizer(num_classes=len(characters) + 1) | |
| checkpoint_load = _load_initial_checkpoint(model, config.initial_checkpoint) | |
| model.to(device) | |
| model.train() | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=config.learning_rate, weight_decay=3e-5) | |
| total_steps = max(1, math.ceil(len(loader) * config.epochs)) | |
| if config.max_steps is not None: | |
| total_steps = min(total_steps, config.max_steps) | |
| backbone_parameter = model.backbone.conv1.conv.weight | |
| backbone_before = backbone_parameter.detach().cpu().clone() | |
| losses: list[float] = [] | |
| optimizer_steps = 0 | |
| last_logits_device = "" | |
| while optimizer_steps < total_steps: | |
| for batch in loader: | |
| images = batch["images"].to(device) | |
| optimizer.zero_grad(set_to_none=True) | |
| logits = model.forward_logits(images) | |
| last_logits_device = logits.device.type | |
| loss = ctc_training_loss( | |
| logits, | |
| batch["targets"], | |
| batch["input_lengths"], | |
| batch["target_lengths"], | |
| ) | |
| if not torch.isfinite(loss).item(): | |
| raise DataValidationError(f"non-finite CTC loss at optimizer step {optimizer_steps + 1}") | |
| loss.backward() | |
| if any( | |
| parameter.grad is not None and not torch.isfinite(parameter.grad).all().item() | |
| for parameter in model.parameters() | |
| ): | |
| raise DataValidationError(f"non-finite gradient at optimizer step {optimizer_steps + 1}") | |
| optimizer.step() | |
| losses.append(float(loss.detach().cpu())) | |
| optimizer_steps += 1 | |
| if optimizer_steps >= total_steps: | |
| break | |
| backbone_after = backbone_parameter.detach().cpu() | |
| backbone_delta = float((backbone_after - backbone_before).abs().max()) | |
| if backbone_delta <= 0: | |
| raise DataValidationError( | |
| "optimizer completed without changing the backbone; check CTC guide detachment/device path" | |
| ) | |
| output = Path(config.output_dir) | |
| output.mkdir(parents=True, exist_ok=True) | |
| checkpoint_path = output / "paddle_mps_checkpoint.pt" | |
| torch.save( | |
| { | |
| "schema_version": 1, | |
| "backend": config.backend, | |
| "source_revision": config.source_revision, | |
| "model_state_dict": {key: value.detach().cpu() for key, value in model.state_dict().items()}, | |
| "optimizer_state_dict": _tree_to_cpu(optimizer.state_dict()), | |
| "characters": characters, | |
| "config": config.to_dict(), | |
| }, | |
| checkpoint_path, | |
| ) | |
| metrics = { | |
| "backend": config.backend, | |
| "source_revision": config.source_revision, | |
| "torch_version": torch.__version__, | |
| "mps_built": torch.backends.mps.is_built(), | |
| "mps_available": torch.backends.mps.is_available(), | |
| "device_requested": config.device, | |
| "device_resolved": device.type, | |
| "model_device": next(model.parameters()).device.type, | |
| "last_logits_device": last_logits_device, | |
| "optimizer_steps": optimizer_steps, | |
| "initial_loss": losses[0], | |
| "final_loss": losses[-1], | |
| "backbone_parameter": "backbone.conv1.conv.weight", | |
| "backbone_parameter_delta_linf": backbone_delta, | |
| "checkpoint_load": checkpoint_load, | |
| "checkpoint_path": str(checkpoint_path), | |
| } | |
| (output / "paddle_mps_training_metrics.json").write_text( | |
| json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| return checkpoint_path | |