File size: 4,397 Bytes
f5d6a7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
"""Write training progress and measured ETA next to the training log."""

import json
import os
import time
from datetime import datetime, timezone
from pathlib import Path

from transformers import TrainerCallback


class TrainingProgressCallback(TrainerCallback):
    def __init__(self):
        self.path = Path(__file__).with_name("training_progress.json")
        self.anchor = None
        self.eval_seconds = None
        self.warmup_end_step = 2
        self.recovery_interval = int(os.environ.get("TRAIN_RECOVERY_SAVE_STEPS", "0"))
        self.last_checkpoint = None

    def write(self, state, phase, **extra):
        if not state.is_world_process_zero:
            return
        record = {
            "phase": phase,
            "pid": os.getpid(),
            "updated_at": datetime.now(timezone.utc).isoformat(),
            "step": state.global_step,
            "total_steps": state.max_steps,
            "epoch": state.epoch,
            "world_size": int(os.environ.get("WORLD_SIZE", "1")),
            "last_checkpoint": self.last_checkpoint,
            **extra,
        }
        temporary = self.path.with_suffix(".tmp")
        temporary.write_text(json.dumps(record, indent=2))
        temporary.replace(self.path)

    def on_train_begin(self, args, state, control, **kwargs):
        self.warmup_end_step = state.global_step + 2
        for record in reversed(state.log_history):
            if "eval_runtime" in record:
                self.eval_seconds = record["eval_runtime"]
                break
        self.write(state, "training")

    def on_step_end(self, args, state, control, **kwargs):
        now = time.monotonic()
        extra = {}
        # Exclude the first two steps, which can include initial kernel compilation.
        if self.anchor is not None:
            start, step = self.anchor
            seconds_per_step = (now - start) / (state.global_step - step)
            remaining = seconds_per_step * (state.max_steps - state.global_step)
            if self.eval_seconds is not None:
                remaining += self.eval_seconds * (
                    state.max_steps // args.eval_steps - state.global_step // args.eval_steps
                )
            extra = {
                "seconds_per_step": seconds_per_step,
                "estimated_seconds_remaining": remaining,
                "estimate_includes_evaluation": self.eval_seconds is not None,
            }
        if state.global_step == self.warmup_end_step:
            self.anchor = (now, state.global_step)
        if self.recovery_interval and (state.global_step == 1 or state.global_step % self.recovery_interval == 0):
            control.should_save = True
        self.write(state, "checkpointing" if control.should_save else "training", **extra)
        return control

    def on_save(self, args, state, control, **kwargs):
        checkpoint = Path(args.output_dir) / f"checkpoint-{state.global_step}"
        if state.is_world_process_zero:
            required = ("trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json")
            if all((checkpoint / name).is_file() for name in required):
                temporary = checkpoint / "recovery_complete.tmp"
                temporary.write_text(json.dumps({"step": state.global_step, "completed_at": datetime.now(timezone.utc).isoformat()}))
                temporary.replace(checkpoint / "recovery_complete.json")
                self.last_checkpoint = str(checkpoint)
        self.write(state, "training")

    def on_evaluate(self, args, state, control, metrics=None, **kwargs):
        self.eval_seconds = (metrics or {}).get("eval_runtime", self.eval_seconds)
        # Start a fresh throughput window so evaluation is not double counted.
        self.anchor = (time.monotonic(), state.global_step)
        self.write(state, "training", metrics=metrics)

    def on_train_end(self, args, state, control, **kwargs):
        self.write(state, "saving_model", best_checkpoint=state.best_model_checkpoint)


def mark_training_complete(trainer):
    callback = TrainingProgressCallback()
    callback.write(
        trainer.state,
        "completed",
        best_checkpoint=trainer.state.best_model_checkpoint,
        best_eval_loss=trainer.state.best_metric,
        final_model=os.path.join(trainer.args.output_dir, "final_model"),
    )