temp / project /training_progress.py
PriyaMishra's picture
Copy model from AjayRangoji/temp
f5d6a7c verified
Raw History Blame Contribute Delete
4.4 kB
"""Write training progress and measured ETA next to the training log."""
import json
import os
import time
from datetime import datetime, timezone
from pathlib import Path
from transformers import TrainerCallback
class TrainingProgressCallback(TrainerCallback):
def __init__(self):
self.path = Path(__file__).with_name("training_progress.json")
self.anchor = None
self.eval_seconds = None
self.warmup_end_step = 2
self.recovery_interval = int(os.environ.get("TRAIN_RECOVERY_SAVE_STEPS", "0"))
self.last_checkpoint = None
def write(self, state, phase, **extra):
if not state.is_world_process_zero:
return
record = {
"phase": phase,
"pid": os.getpid(),
"updated_at": datetime.now(timezone.utc).isoformat(),
"step": state.global_step,
"total_steps": state.max_steps,
"epoch": state.epoch,
"world_size": int(os.environ.get("WORLD_SIZE", "1")),
"last_checkpoint": self.last_checkpoint,
**extra,
}
temporary = self.path.with_suffix(".tmp")
temporary.write_text(json.dumps(record, indent=2))
temporary.replace(self.path)
def on_train_begin(self, args, state, control, **kwargs):
self.warmup_end_step = state.global_step + 2
for record in reversed(state.log_history):
if "eval_runtime" in record:
self.eval_seconds = record["eval_runtime"]
break
self.write(state, "training")
def on_step_end(self, args, state, control, **kwargs):
now = time.monotonic()
extra = {}
# Exclude the first two steps, which can include initial kernel compilation.
if self.anchor is not None:
start, step = self.anchor
seconds_per_step = (now - start) / (state.global_step - step)
remaining = seconds_per_step * (state.max_steps - state.global_step)
if self.eval_seconds is not None:
remaining += self.eval_seconds * (
state.max_steps // args.eval_steps - state.global_step // args.eval_steps
)
extra = {
"seconds_per_step": seconds_per_step,
"estimated_seconds_remaining": remaining,
"estimate_includes_evaluation": self.eval_seconds is not None,
}
if state.global_step == self.warmup_end_step:
self.anchor = (now, state.global_step)
if self.recovery_interval and (state.global_step == 1 or state.global_step % self.recovery_interval == 0):
control.should_save = True
self.write(state, "checkpointing" if control.should_save else "training", **extra)
return control
def on_save(self, args, state, control, **kwargs):
checkpoint = Path(args.output_dir) / f"checkpoint-{state.global_step}"
if state.is_world_process_zero:
required = ("trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json")
if all((checkpoint / name).is_file() for name in required):
temporary = checkpoint / "recovery_complete.tmp"
temporary.write_text(json.dumps({"step": state.global_step, "completed_at": datetime.now(timezone.utc).isoformat()}))
temporary.replace(checkpoint / "recovery_complete.json")
self.last_checkpoint = str(checkpoint)
self.write(state, "training")
def on_evaluate(self, args, state, control, metrics=None, **kwargs):
self.eval_seconds = (metrics or {}).get("eval_runtime", self.eval_seconds)
# Start a fresh throughput window so evaluation is not double counted.
self.anchor = (time.monotonic(), state.global_step)
self.write(state, "training", metrics=metrics)
def on_train_end(self, args, state, control, **kwargs):
self.write(state, "saving_model", best_checkpoint=state.best_model_checkpoint)
def mark_training_complete(trainer):
callback = TrainingProgressCallback()
callback.write(
trainer.state,
"completed",
best_checkpoint=trainer.state.best_model_checkpoint,
best_eval_loss=trainer.state.best_metric,
final_model=os.path.join(trainer.args.output_dir, "final_model"),
)