Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
File size: 4,397 Bytes
f5d6a7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | """Write training progress and measured ETA next to the training log."""
import json
import os
import time
from datetime import datetime, timezone
from pathlib import Path
from transformers import TrainerCallback
class TrainingProgressCallback(TrainerCallback):
def __init__(self):
self.path = Path(__file__).with_name("training_progress.json")
self.anchor = None
self.eval_seconds = None
self.warmup_end_step = 2
self.recovery_interval = int(os.environ.get("TRAIN_RECOVERY_SAVE_STEPS", "0"))
self.last_checkpoint = None
def write(self, state, phase, **extra):
if not state.is_world_process_zero:
return
record = {
"phase": phase,
"pid": os.getpid(),
"updated_at": datetime.now(timezone.utc).isoformat(),
"step": state.global_step,
"total_steps": state.max_steps,
"epoch": state.epoch,
"world_size": int(os.environ.get("WORLD_SIZE", "1")),
"last_checkpoint": self.last_checkpoint,
**extra,
}
temporary = self.path.with_suffix(".tmp")
temporary.write_text(json.dumps(record, indent=2))
temporary.replace(self.path)
def on_train_begin(self, args, state, control, **kwargs):
self.warmup_end_step = state.global_step + 2
for record in reversed(state.log_history):
if "eval_runtime" in record:
self.eval_seconds = record["eval_runtime"]
break
self.write(state, "training")
def on_step_end(self, args, state, control, **kwargs):
now = time.monotonic()
extra = {}
# Exclude the first two steps, which can include initial kernel compilation.
if self.anchor is not None:
start, step = self.anchor
seconds_per_step = (now - start) / (state.global_step - step)
remaining = seconds_per_step * (state.max_steps - state.global_step)
if self.eval_seconds is not None:
remaining += self.eval_seconds * (
state.max_steps // args.eval_steps - state.global_step // args.eval_steps
)
extra = {
"seconds_per_step": seconds_per_step,
"estimated_seconds_remaining": remaining,
"estimate_includes_evaluation": self.eval_seconds is not None,
}
if state.global_step == self.warmup_end_step:
self.anchor = (now, state.global_step)
if self.recovery_interval and (state.global_step == 1 or state.global_step % self.recovery_interval == 0):
control.should_save = True
self.write(state, "checkpointing" if control.should_save else "training", **extra)
return control
def on_save(self, args, state, control, **kwargs):
checkpoint = Path(args.output_dir) / f"checkpoint-{state.global_step}"
if state.is_world_process_zero:
required = ("trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json")
if all((checkpoint / name).is_file() for name in required):
temporary = checkpoint / "recovery_complete.tmp"
temporary.write_text(json.dumps({"step": state.global_step, "completed_at": datetime.now(timezone.utc).isoformat()}))
temporary.replace(checkpoint / "recovery_complete.json")
self.last_checkpoint = str(checkpoint)
self.write(state, "training")
def on_evaluate(self, args, state, control, metrics=None, **kwargs):
self.eval_seconds = (metrics or {}).get("eval_runtime", self.eval_seconds)
# Start a fresh throughput window so evaluation is not double counted.
self.anchor = (time.monotonic(), state.global_step)
self.write(state, "training", metrics=metrics)
def on_train_end(self, args, state, control, **kwargs):
self.write(state, "saving_model", best_checkpoint=state.best_model_checkpoint)
def mark_training_complete(trainer):
callback = TrainingProgressCallback()
callback.write(
trainer.state,
"completed",
best_checkpoint=trainer.state.best_model_checkpoint,
best_eval_loss=trainer.state.best_metric,
final_model=os.path.join(trainer.args.output_dir, "final_model"),
)
|