"""Write training progress and measured ETA next to the training log.""" import json import os import time from datetime import datetime, timezone from pathlib import Path from transformers import TrainerCallback class TrainingProgressCallback(TrainerCallback): def __init__(self): self.path = Path(__file__).with_name("training_progress.json") self.anchor = None self.eval_seconds = None self.warmup_end_step = 2 self.recovery_interval = int(os.environ.get("TRAIN_RECOVERY_SAVE_STEPS", "0")) self.last_checkpoint = None def write(self, state, phase, **extra): if not state.is_world_process_zero: return record = { "phase": phase, "pid": os.getpid(), "updated_at": datetime.now(timezone.utc).isoformat(), "step": state.global_step, "total_steps": state.max_steps, "epoch": state.epoch, "world_size": int(os.environ.get("WORLD_SIZE", "1")), "last_checkpoint": self.last_checkpoint, **extra, } temporary = self.path.with_suffix(".tmp") temporary.write_text(json.dumps(record, indent=2)) temporary.replace(self.path) def on_train_begin(self, args, state, control, **kwargs): self.warmup_end_step = state.global_step + 2 for record in reversed(state.log_history): if "eval_runtime" in record: self.eval_seconds = record["eval_runtime"] break self.write(state, "training") def on_step_end(self, args, state, control, **kwargs): now = time.monotonic() extra = {} # Exclude the first two steps, which can include initial kernel compilation. if self.anchor is not None: start, step = self.anchor seconds_per_step = (now - start) / (state.global_step - step) remaining = seconds_per_step * (state.max_steps - state.global_step) if self.eval_seconds is not None: remaining += self.eval_seconds * ( state.max_steps // args.eval_steps - state.global_step // args.eval_steps ) extra = { "seconds_per_step": seconds_per_step, "estimated_seconds_remaining": remaining, "estimate_includes_evaluation": self.eval_seconds is not None, } if state.global_step == self.warmup_end_step: self.anchor = (now, state.global_step) if self.recovery_interval and (state.global_step == 1 or state.global_step % self.recovery_interval == 0): control.should_save = True self.write(state, "checkpointing" if control.should_save else "training", **extra) return control def on_save(self, args, state, control, **kwargs): checkpoint = Path(args.output_dir) / f"checkpoint-{state.global_step}" if state.is_world_process_zero: required = ("trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json") if all((checkpoint / name).is_file() for name in required): temporary = checkpoint / "recovery_complete.tmp" temporary.write_text(json.dumps({"step": state.global_step, "completed_at": datetime.now(timezone.utc).isoformat()})) temporary.replace(checkpoint / "recovery_complete.json") self.last_checkpoint = str(checkpoint) self.write(state, "training") def on_evaluate(self, args, state, control, metrics=None, **kwargs): self.eval_seconds = (metrics or {}).get("eval_runtime", self.eval_seconds) # Start a fresh throughput window so evaluation is not double counted. self.anchor = (time.monotonic(), state.global_step) self.write(state, "training", metrics=metrics) def on_train_end(self, args, state, control, **kwargs): self.write(state, "saving_model", best_checkpoint=state.best_model_checkpoint) def mark_training_complete(trainer): callback = TrainingProgressCallback() callback.write( trainer.state, "completed", best_checkpoint=trainer.state.best_model_checkpoint, best_eval_loss=trainer.state.best_metric, final_model=os.path.join(trainer.args.output_dir, "final_model"), )