Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
Download project/training_progress.py from PriyaMishra/temp: direct link, hf CLI and curl.
- Browser
- Download file 4.4 kB
-
https://huggingface.co/PriyaMishra/temp/resolve/main/project/training_progress.py
- Command line
-
hf download hf://PriyaMishra/temp/project/training_progress.py
-
curl -L -o training_progress.py https://huggingface.co/PriyaMishra/temp/resolve/main/project/training_progress.py
4.4 kB
| """Write training progress and measured ETA next to the training log.""" | |
| import json | |
| import os | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from transformers import TrainerCallback | |
| class TrainingProgressCallback(TrainerCallback): | |
| def __init__(self): | |
| self.path = Path(__file__).with_name("training_progress.json") | |
| self.anchor = None | |
| self.eval_seconds = None | |
| self.warmup_end_step = 2 | |
| self.recovery_interval = int(os.environ.get("TRAIN_RECOVERY_SAVE_STEPS", "0")) | |
| self.last_checkpoint = None | |
| def write(self, state, phase, **extra): | |
| if not state.is_world_process_zero: | |
| return | |
| record = { | |
| "phase": phase, | |
| "pid": os.getpid(), | |
| "updated_at": datetime.now(timezone.utc).isoformat(), | |
| "step": state.global_step, | |
| "total_steps": state.max_steps, | |
| "epoch": state.epoch, | |
| "world_size": int(os.environ.get("WORLD_SIZE", "1")), | |
| "last_checkpoint": self.last_checkpoint, | |
| **extra, | |
| } | |
| temporary = self.path.with_suffix(".tmp") | |
| temporary.write_text(json.dumps(record, indent=2)) | |
| temporary.replace(self.path) | |
| def on_train_begin(self, args, state, control, **kwargs): | |
| self.warmup_end_step = state.global_step + 2 | |
| for record in reversed(state.log_history): | |
| if "eval_runtime" in record: | |
| self.eval_seconds = record["eval_runtime"] | |
| break | |
| self.write(state, "training") | |
| def on_step_end(self, args, state, control, **kwargs): | |
| now = time.monotonic() | |
| extra = {} | |
| # Exclude the first two steps, which can include initial kernel compilation. | |
| if self.anchor is not None: | |
| start, step = self.anchor | |
| seconds_per_step = (now - start) / (state.global_step - step) | |
| remaining = seconds_per_step * (state.max_steps - state.global_step) | |
| if self.eval_seconds is not None: | |
| remaining += self.eval_seconds * ( | |
| state.max_steps // args.eval_steps - state.global_step // args.eval_steps | |
| ) | |
| extra = { | |
| "seconds_per_step": seconds_per_step, | |
| "estimated_seconds_remaining": remaining, | |
| "estimate_includes_evaluation": self.eval_seconds is not None, | |
| } | |
| if state.global_step == self.warmup_end_step: | |
| self.anchor = (now, state.global_step) | |
| if self.recovery_interval and (state.global_step == 1 or state.global_step % self.recovery_interval == 0): | |
| control.should_save = True | |
| self.write(state, "checkpointing" if control.should_save else "training", **extra) | |
| return control | |
| def on_save(self, args, state, control, **kwargs): | |
| checkpoint = Path(args.output_dir) / f"checkpoint-{state.global_step}" | |
| if state.is_world_process_zero: | |
| required = ("trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json") | |
| if all((checkpoint / name).is_file() for name in required): | |
| temporary = checkpoint / "recovery_complete.tmp" | |
| temporary.write_text(json.dumps({"step": state.global_step, "completed_at": datetime.now(timezone.utc).isoformat()})) | |
| temporary.replace(checkpoint / "recovery_complete.json") | |
| self.last_checkpoint = str(checkpoint) | |
| self.write(state, "training") | |
| def on_evaluate(self, args, state, control, metrics=None, **kwargs): | |
| self.eval_seconds = (metrics or {}).get("eval_runtime", self.eval_seconds) | |
| # Start a fresh throughput window so evaluation is not double counted. | |
| self.anchor = (time.monotonic(), state.global_step) | |
| self.write(state, "training", metrics=metrics) | |
| def on_train_end(self, args, state, control, **kwargs): | |
| self.write(state, "saving_model", best_checkpoint=state.best_model_checkpoint) | |
| def mark_training_complete(trainer): | |
| callback = TrainingProgressCallback() | |
| callback.write( | |
| trainer.state, | |
| "completed", | |
| best_checkpoint=trainer.state.best_model_checkpoint, | |
| best_eval_loss=trainer.state.best_metric, | |
| final_model=os.path.join(trainer.args.output_dir, "final_model"), | |
| ) | |