code/wavecurriculum_run/train/train_and_evaluate.py

# coding=utf-8
# Copyright 2024 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This file contains function that coordinates the training and evaluation of the model."""

import functools
import json
import math
import os
import signal
import socket
import time
import traceback

from absl import logging
from clu import metric_writers
from flax import jax_utils
from flax import linen as nn
from flax.training import checkpoints
import jax
from jax import random
import jax.numpy as jnp
import numpy as np
import tensorflow as tf
import wandb

from train import data
from train import evaluater
from train import model
from train import trainer



def log_hyperparams_tb(
    config, model_config, initial_variables, tf_summary_writer
):
    """Log hyperparameters to TensorBoard.

    Args:
        config: experiment's ConfigDict
        model_config: model's ConfigDict
        initial_variables: initial hyperparameter values
        tf_summary_writer: SummaryWriter object.

    Returns:
        The SummaryWriter object and the config.
    """
    # Calculate the total number of model parameters
    config.num_model_parameters = sum(
        x.size for x in jax.tree_util.tree_leaves(initial_variables)
    )

    # Convert hyperparameters to tensors
    config_hyperparameters = [
        tf.convert_to_tensor([k, str(v)]) for k, v in config.items()
    ]
    model_config_hyperparameters = [
        tf.convert_to_tensor([k, str(v)])
        for k, v in model_config.__dict__.items()
    ]

    # Log model hyperparameters to TensorBoard
    with tf_summary_writer.as_default():
        tf.summary.text(
            "Model hyperparameters", tf.stack(model_config_hyperparameters), step=0
        )
        tf.summary.text(
            "Config hyperparameters", tf.stack(config_hyperparameters), step=0
        )

    return tf_summary_writer, config




def _write_json(path, payload):
    """Atomic JSON write so a sidecar never reads a half-written file."""
    tmp = path + ".tmp"
    with open(tmp, "w") as f:
        json.dump(payload, f, indent=2, sort_keys=True)
        f.write("\n")
    os.replace(tmp, path)


def _write_heartbeat(workdir, payload):
    payload = dict(payload)
    payload.setdefault("host", socket.gethostname())
    payload.setdefault("job_id", os.environ.get("SLURM_JOB_ID", ""))
    payload["unix_time"] = time.time()
    payload["iso_time"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
    slim = {k: payload[k] for k in
            ("event", "step", "stage", "loss",
             # val_acc = P(digit not in the stage-t candidate set)
             "val_acc", "puzzle_acc",
             # location order, tolerant to ties within a propagation wave
             "loc_wave", "loc_coverage", "loc_lcs", "loc_dup", "loc_acc",
             # superposition: excess 0 == uniform over the candidate set
             "val_excess", "val_mass", "val_spread",
             "val_excess_multi", "val_mass_multi", "val_out_multi",
             "val_kl_multi",
             "last_ckpt_step", "error")
            if k in payload}
    _write_json(os.path.join(workdir, "heartbeat.json"), slim)


def train_and_evaluate(config, workdir):
    """The training and evaluation loops for the model.

    Args:
        config: experiment's config dictionary.
        workdir: directory to use for logging.
    """
    # Orbax checkpointing requires an absolute path.
    workdir = os.path.abspath(workdir)

    logging.info("Creating training and evaluator dataset iterator")
    curriculum = data.CurriculumState(
        stage=int(getattr(config, "curriculum_start_stage", 1)),
        max_stage=int(getattr(config, "curriculum_max_stage", 6)))
    train_data_iter = data.create_iter(
        config, config.minibatch_size, train=True, curriculum=curriculum)
    eval_data_iter = data.create_iter(config, config.minibatch_size, train=False)

    logging.info("Finished creating training dataset iterator")

    model_config = model.TransformerConfig(
        dtype=config.dtype,
        vocab_size=config.vocab_size,
        seq_len=config.seq_len,
        num_heads=config.num_heads,
        num_layers=config.num_layers,
        emb_dim=config.emb_dim,
        qkv_dim=config.qkv_dim,
        mlp_dim=config.mlp_dim,
        dropout_rate=config.dropout_rate,
        attention_dropout_rate=config.attention_dropout_rate,
        deterministic=False,
        num_latent_slots=int(config.num_latent_slots),
        inject_latents=bool(int(getattr(config, "recurrent_latent", 1))),
    )

    logging.info("train_config: %s", str(model_config.__dict__))
    print(str(model_config.__dict__), flush=True)

    rng = jax.random.PRNGKey(config.seed)
    rng, init_rng, inference_rng = random.split(rng, num=3)

    # Initialize the model and get initial variables. Dummy latent arguments
    # are provided so the latent projector parameters are created at init.
    rng, dropout_rng = jax.random.split(rng)
    input_shape = (config.minibatch_size, config.seq_len)
    net = model.TransformerLMHeadModel(model_config)
    rng_keys = {"params": init_rng, "dropout": dropout_rng}
    K = int(config.num_latent_slots)
    dummy_latents = jnp.zeros(
        (config.minibatch_size, K, config.emb_dim), model_config.dtype)
    dummy_positions = jnp.zeros((config.minibatch_size, K), jnp.int32)
    dummy_active = jnp.zeros((config.minibatch_size, K), bool)
    sample_out, initial_variables = jax.jit(
        net.init_with_output
        )(rng_keys, jnp.ones(input_shape, jnp.int32), dummy_latents,
          dummy_positions, dummy_active)
    
    state, lr_scheduler_fn = trainer.get_state(config, net, initial_variables)
    # Resume-and-extend support: when resuming, start the training loop at the
    # restored optimizer step (not 0) so a larger config.max_steps continues the
    # cosine LR schedule cleanly instead of re-running steps or overshooting.
    start_step = 0
    if config.resume_training:
        state = checkpoints.restore_checkpoint(config.ckpt_loc, state)
        start_step = int(state.step)
        print("----------Restored model from", config.ckpt_loc,
              f"at step {start_step}-----------")

    writer = metric_writers.create_default_writer(
        workdir, asynchronous=False, just_logging=(jax.process_index() > 0))
    tf_summary_writer = tf.summary.create_file_writer(workdir)

    logging.info("config: %s", str(config.__dict__))
    state = jax_utils.replicate(state)

    dropout_rngs = jax.random.split(rng, jax.local_device_count())

    def make_p_train_step(num_passes):
        return jax.pmap(
            functools.partial(
                trainer.train_step,
                config=model_config,
                hyperparams=config,
                learning_rate_fn=lr_scheduler_fn,
                num_passes=num_passes),
            axis_name="batch",
            donate_argnums=(0,))

    # num_passes = latent recurrence depth for this stage, capped by the number
    # of latent slots (K=0 -> 0 passes -> no-latent control baseline).
    # When recurrence is disabled (control), force 0 passes so no hidden state
    # is ever fed back: the slots become static, independent per-stage readouts.
    recurrent = bool(int(getattr(config, "recurrent_latent", 1)))
    passes_per_stage = int(getattr(config, "passes_per_stage", 1))
    def passes_for(stage):
        return min(passes_per_stage * stage, K) if recurrent else 0
    p_train_step = make_p_train_step(passes_for(curriculum.stage))

    p_eval_step = jax.pmap(functools.partial(evaluater.eval_step,
                                             config=model_config.replace(deterministic=True)),
                                             axis_name="batch")
    
    hooks, report_progress, train_metrics = trainer.get_metrics_report_progress(
      config, workdir, writer)

    tf_summary_writer, config = log_hyperparams_tb(
        config, model_config, initial_variables, tf_summary_writer
    )

    # Instance-arm promotion is THREE gates, all required, no patience/plateau:
    #   val_mass_multi  >= promote_mass_threshold    (support inside S)
    #   val_spread      >= promote_spread_threshold  (uniform over S)
    #   val_hbound      >= promote_hbound_threshold  (H(p) <= log|S| per cell)
    # S is the raw per-cell candidate set from the wave mask, not the filtered
    # instance support after constraint solving.
    promote_mass_threshold = float(getattr(config, "promote_acc_threshold", 0.85))
    promote_spread_threshold = float(
        getattr(config, "promote_spread_threshold", 0.85))
    promote_hbound_threshold = float(
        getattr(config, "promote_hbound_threshold", 0.0))
    # When set, this is the ONLY value gate: CE(q||p) <= log|S| already implies
    # both "mass is inside S" and "no candidate was dropped", so mass*/spread*/
    # hbound become diagnostics. See _inst_ce_stats in evaluater.py.
    promote_ceq_threshold = float(getattr(config, "promote_ceq_threshold", 0.0))
    # Applies to loc_wave (order up to wave ties), not the positional loc_acc.
    # 0 disables the gate.
    promote_loc_threshold = float(getattr(config, "promote_loc_threshold", 0.0))
    promote_patience = int(getattr(config, "promote_patience_steps", 8000))
    min_stage_steps = int(getattr(config, "min_stage_steps", 2000))
    plateau_steps = int(getattr(config, "plateau_steps", 0))
    plateau_delta = float(getattr(config, "plateau_delta", 0.005))
    instance_mode = bool(getattr(config, "instance_dir", None))
    per_stage_inset = {}
    per_stage_vexcess, per_stage_vmass, per_stage_vspread = {}, {}, {}
    per_stage_vexc_m, per_stage_vmass_m, per_stage_vkl_m = {}, {}, {}
    per_stage_vhb, per_stage_vhgap, per_stage_vhb_puz = {}, {}, {}
    per_stage_ceq, per_stage_hq, per_stage_ceq_ok = {}, {}, {}
    per_stage_ceq_gap = {}
    # Best frontier-depth accuracy seen in the current stage, and the step it
    # was last improved: the plateau detector's state.
    stage_best_acc = -1.0
    stage_best_step = start_step
    ckpt_keep = int(getattr(config, "ckpt_keep", 100))
    # Protected directory for per-stage checkpoints (never rolled off).
    stage_ckpt_dir = os.path.join(workdir, "stage_ckpts")
    stage_started_at = start_step
    last_ckpt_step = start_step
    stop_requested = {"flag": False}

    def _on_stop(signum, _frame):
        print(f"[signal] {signum} received; will save and exit after this step",
              flush=True)
        stop_requested["flag"] = True

    signal.signal(signal.SIGTERM, _on_stop)
    signal.signal(signal.SIGINT, _on_stop)
    _write_heartbeat(workdir, {
        "event": "start",
        "step": start_step,
        "stage": curriculum.stage,
        "max_steps": int(config.max_steps),
    })

    with metric_writers.ensure_flushes(writer):
        for step in range(start_step, config.max_steps):
            if step%10000 == 0:
                print("Step:", step, flush=True)

            state, metrics = trainer.train_one_step(p_train_step, config, state, 
                                                    step, dropout_rngs, train_data_iter)
            
            for h in hooks:
                h(step)

            if math.isnan(metrics["loss"][0]):
                print("The loss function became nan: This might be due to the choice of hyperparameters.")
                _write_heartbeat(workdir, {
                    "event": "nan_loss", "step": step, "stage": curriculum.stage,
                })
                break

            if step % config.eval_every_steps == 0:
                try:
                    eval_metrics = evaluater.get_eval_metrics(
                        state, eval_data_iter, p_eval_step, config)
                except Exception:
                    traceback.print_exc()
                    _write_heartbeat(workdir, {
                        "event": "eval_error",
                        "step": step,
                        "stage": curriculum.stage,
                        "loss": float(metrics["loss"].mean()),
                        "ce": float(metrics["ce_loss"].mean()),
                        "error": traceback.format_exc()[-2000:],
                    })
                    print("[eval] failed; skipping this eval and continuing",
                          flush=True)
                    eval_metrics = None
                if eval_metrics is not None:
                    per_level = eval_metrics.pop("per_level_acc")
                    per_slot = eval_metrics.pop("per_slot_acc", {})
                    per_slot_ch = eval_metrics.pop("per_slot_acc_changed", {})
                    per_stage_inset = eval_metrics.pop("per_stage_inset_acc", {})
                    per_stage_vexcess = eval_metrics.pop(
                        "per_stage_val_excess", {})
                    per_stage_vmass = eval_metrics.pop("per_stage_val_mass", {})
                    per_stage_vspread = eval_metrics.pop(
                        "per_stage_val_spread", {})
                    per_stage_vexc_m = eval_metrics.pop(
                        "per_stage_val_excess_multi", {})
                    per_stage_vmass_m = eval_metrics.pop(
                        "per_stage_val_mass_multi", {})
                    per_stage_vkl_m = eval_metrics.pop(
                        "per_stage_val_kl_multi", {})
                    per_stage_vhb = eval_metrics.pop(
                        "per_stage_val_hbound", {})
                    per_stage_vhgap = eval_metrics.pop(
                        "per_stage_val_hgap", {})
                    per_stage_vhb_puz = eval_metrics.pop(
                        "per_stage_val_hbound_puz", {})
                    per_stage_ceq = eval_metrics.pop("per_stage_val_ceq", {})
                    per_stage_hq = eval_metrics.pop("per_stage_val_hq", {})
                    per_stage_ceq_ok = eval_metrics.pop(
                        "per_stage_val_ceq_ok", {})
                    per_stage_ceq_gap = eval_metrics.pop(
                        "per_stage_val_ceq_gap", {})
                    eval_metrics.pop("per_stage_val_ceq_puz", {})
                    eval_metrics.pop("per_stage_val_ce", {})
                    eval_metrics.pop("per_stage_val_floor", {})
                    per_bin = eval_metrics.pop("per_bin_acc", {})
                    def _m(key):
                        v = eval_metrics.get(key, [])
                        return round(float(np.mean(v)), 4) if len(v) else -1.0
                    def _s(d):
                        return round(float(d.get(curriculum.stage, -1.0)), 4)
                    # val_acc = P(digit not in stage-t candidate set),
                    # teacher-forced so location cannot contaminate it.
                    # 0 = all mass inside S; chance at stage 0 is ~0.59.
                    mass = _s(per_stage_vmass)
                    val_out = round(1.0 - mass, 4) if mass >= 0 else -1.0
                    mass_m = _s(per_stage_vmass_m)
                    val_out_m = round(1.0 - mass_m, 4) if mass_m >= 0 else -1.0
                    print(step,
                          "stage", curriculum.stage,
                          "loss", round(float(metrics["loss"].mean()), 4),
                          "mass*", mass_m,
                          "spread*", _s(per_stage_vspread),
                          "hbound", _s(per_stage_vhb),
                          # ceq must sit under log|S| and above H(q); ceq_ok is
                          # the fraction of cells that manage it.
                          "ceq", _s(per_stage_ceq),
                          "H(q)", _s(per_stage_hq),
                          "ceq_ok", _s(per_stage_ceq_ok),
                          "val_acc", val_out_m,
                          flush=True)
                    _write_heartbeat(workdir, {
                        "event": "eval",
                        "step": int(step),
                        "stage": int(curriculum.stage),
                        "loss": round(float(metrics["loss"].mean()), 6),
                        "ce": round(float(metrics["ce_loss"].mean()), 6),
                        "val_acc": val_out_m,
                        "puzzle_acc": _m("acc_complete_puzzle"),
                        # Permutation-tolerant location signals; loc_acc is the
                        # old positional match, kept only for continuity.
                        "loc_wave": _m("loc_wave"),
                        "loc_coverage": _m("loc_coverage"),
                        "loc_lcs": _m("loc_lcs"),
                        "loc_dup": _m("loc_dup"),
                        "loc_acc": _m("loc_acc"),
                        # Superposition: 0 excess == uniform over the stage's
                        # candidate set; spread 1.0 == no mode collapse.
                        "val_excess": _s(per_stage_vexcess),
                        "val_mass": _s(per_stage_vmass),
                        "val_spread": _s(per_stage_vspread),
                        # |S|>=2 cells only: excess_multi == log(1/mass_multi)
                        # + kl_multi, so it is the single number that falls only
                        # when leakage and non-uniformity both fall.
                        "val_excess_multi": _s(per_stage_vexc_m),
                        "val_mass_multi": mass_m,
                        "val_out_multi": val_out_m,
                        "val_kl_multi": _s(per_stage_vkl_m),
                        # Entropy ceiling H(p) <= log|S|: fraction of |S|>=2
                        # cells that satisfy it, the mean margin in nats, and
                        # the stricter all-cells-in-the-puzzle view.
                        "val_hbound": _s(per_stage_vhb),
                        "val_hgap": _s(per_stage_vhgap),
                        "val_hbound_puz": _s(per_stage_vhb_puz),
                        # The promotion criterion and the two numbers that
                        # bracket it: H(q) <= ceq should hold, ceq <= log|S| is
                        # the gate, and ceq_gap is the signed slack.
                        "val_ceq": _s(per_stage_ceq),
                        "val_hq": _s(per_stage_hq),
                        "val_ceq_ok": _s(per_stage_ceq_ok),
                        "val_ceq_gap": _s(per_stage_ceq_gap),
                        "last_ckpt_step": int(last_ckpt_step),
                    })
                    with tf_summary_writer.as_default():
                        tf.summary.scalar("loss", metrics["loss"].mean(), step=step)
                        tf.summary.scalar("ce_loss", metrics["ce_loss"].mean(), step=step)
                        tf.summary.scalar("aux_bce_loss", metrics["aux_loss"].mean(), step=step)
                        tf.summary.scalar(
                            "learning rate", metrics["learning_rate"].mean(), step=step
                            )
                        tf.summary.scalar("curriculum_stage", curriculum.stage, step=step)

                        log_dict = {'loss': metrics["loss"].mean(), 'learning rate': metrics["learning_rate"].mean()}

                        for key in eval_metrics.keys():
                            vals = eval_metrics[key]
                            if not vals:
                                continue
                            tf.summary.scalar(
                                "eval_" + key, np.array(vals).mean(), step=step
                            )
                            log_dict[ "eval_" + key ] = np.array(vals).mean()

                        for lvl, v in per_level.items():
                            if v >= 0:
                                tf.summary.scalar(f"eval_acc_level_{lvl}", v, step=step)
                                log_dict[f"eval_acc_level_{lvl}"] = v

                        for s, v in per_slot.items():
                            if v >= 0:
                                tf.summary.scalar(f"eval_cand_depth_{s}", v, step=step)
                                log_dict[f"eval_cand_depth_{s}"] = v
                        for s, v in per_slot_ch.items():
                            if v >= 0:
                                tf.summary.scalar(f"eval_cand_depth_chg_{s}", v, step=step)

                        if config.use_wandb: wandb.log(log_dict, step=step)

                    # ---- Curriculum promotion ----
                    # Instance arm: THREE gates, all required, no plateau, no
                    # patience. The model must actually learn the current stage's
                    # representation before the next wave is introduced.
                    #   1. val_mass_multi >= mass_bar: support is inside S
                    #   2. val_spread     >= spread_bar: support is uniform on S
                    #   3. val_hbound     >= hbound_bar: H(p) <= log|S| on that
                    #      fraction of cells. Checked per CELL and then counted,
                    #      because the inequality is per cell: comparing mean
                    #      H(p) to mean log|S| would let comfortable cells mask
                    #      violating ones. Catches diffuse leak that mass* is
                    #      blind to, since spreading a fixed leak over many
                    #      illegal digits costs mass the same but entropy more.
                    # S is the raw wave-solver candidate set at that cell, NOT
                    # the support of the filtered instances after constraints.
                    rounds_curric = str(
                        getattr(config, "data_curriculum", "none")) == "rounds"
                    has_frontier = int(config.num_latent_slots) > 0 or rounds_curric
                    if has_frontier and curriculum.stage < curriculum.max_stage:
                        t = curriculum.stage
                        loc_now = _m("loc_wave")
                        mass_m = float(per_stage_vmass_m.get(t, -1.0))
                        spread_now = float(per_stage_vspread.get(t, -1.0))
                        hbound_now = float(per_stage_vhb.get(t, -1.0))
                        if instance_mode:
                            frontier_acc = mass_m
                        elif int(config.num_latent_slots) > 0:
                            frontier_acc = per_slot_ch.get(t, -1.0)
                            if frontier_acc < 0:
                                frontier_acc = per_slot.get(t, -1.0)
                        else:
                            frontier_acc = per_bin.get(t, -1.0)
                            if frontier_acc < 0:
                                seen = [v for b, v in per_bin.items()
                                        if b <= t and v >= 0]
                                frontier_acc = (float(np.mean(seen)) if seen
                                                else -1.0)
                        steps_in_stage = step - stage_started_at
                        measured = frontier_acc >= 0
                        if measured and frontier_acc > stage_best_acc + plateau_delta:
                            stage_best_acc = frontier_acc
                            stage_best_step = step
                        loc_ready = (promote_loc_threshold <= 0.0
                                     or loc_now >= promote_loc_threshold)
                        ceq_ok_now = float(per_stage_ceq_ok.get(t, -1.0))
                        if instance_mode and promote_ceq_threshold > 0.0:
                            # Single criterion. CE(q||p) <= log|S| is violated by
                            # leak and by collapse alike, so the mass and spread
                            # bars would only add mutually infeasible constraints
                            # on top of it (see probe_entropy_ceiling.py).
                            hit_threshold = (ceq_ok_now >= 0 and loc_ready
                                             and ceq_ok_now >= promote_ceq_threshold)
                            stalled = False
                            patience_over = False
                        elif instance_mode:
                            mass_ready = mass_m >= promote_mass_threshold
                            spread_ready = spread_now >= promote_spread_threshold
                            hbound_ready = (promote_hbound_threshold <= 0.0
                                            or hbound_now >= promote_hbound_threshold)
                            hit_threshold = (measured and loc_ready and mass_ready
                                             and spread_ready and hbound_ready)
                            # No plateau, no patience: stay on this stage until
                            # both superposition gates clear.
                            stalled = False
                            patience_over = False
                        else:
                            hit_threshold = (measured and loc_ready
                                             and frontier_acc >= promote_mass_threshold)
                            stalled = (measured and loc_ready and plateau_steps > 0
                                       and (step - stage_best_step) >= plateau_steps)
                            patience_over = (promote_patience > 0
                                             and steps_in_stage >= promote_patience)
                        if steps_in_stage >= min_stage_steps and (
                                hit_threshold or stalled or patience_over):
                            reason = ("threshold" if hit_threshold
                                      else "plateau" if stalled else "patience")
                            curriculum.stage += 1
                            stage_started_at = step
                            stage_best_acc = -1.0
                            stage_best_step = step
                            p_train_step = make_p_train_step(passes_for(curriculum.stage))
                            what = ("round-bin" if int(config.num_latent_slots) == 0
                                    else "depth")
                            if instance_mode and promote_ceq_threshold > 0.0:
                                extra = (f" ceq_ok={ceq_ok_now:.3f}"
                                         f" ceq={float(per_stage_ceq.get(t, -1)):.3f}"
                                         f" H(q)={float(per_stage_hq.get(t, -1)):.3f}")
                            elif instance_mode:
                                extra = (f" mass*={mass_m:.3f}"
                                         f" spread*={spread_now:.3f}"
                                         f" hbound={hbound_now:.3f}")
                            else:
                                extra = f" acc={frontier_acc:.3f}"
                            print(f"[curriculum] step {step}: promote to stage "
                                  f"{curriculum.stage} ({reason}; graduated {what} "
                                  f"{t}{extra} after {steps_in_stage} steps); "
                                  f"latent passes={curriculum.stage}, "
                                  f"pool/snapshots now 1..{curriculum.stage}",
                                  flush=True)
                            if config.save_checkpoint:
                                unrep_state = jax_utils.unreplicate(state)
                                # Rolling checkpoint in the main workdir.
                                checkpoints.save_checkpoint_multiprocess(
                                    workdir, unrep_state, step,
                                    keep=ckpt_keep, overwrite=True)
                                # Protected copy: the model *entering* this stage,
                                # kept permanently under stage_ckpts/ (never rolled
                                # off), so every stage's checkpoint survives.
                                checkpoints.save_checkpoint_multiprocess(
                                    stage_ckpt_dir, unrep_state, step,
                                    keep=100, overwrite=True,
                                    prefix=f"stage{curriculum.stage}_")
                                last_ckpt_step = step
                                _write_json(os.path.join(workdir, "ckpt_ready.json"), {
                                    "event": "stage",
                                    "step": int(step),
                                    "stage": int(curriculum.stage),
                                    "reason": reason,
                                    "workdir": workdir,
                                    "stage_ckpt_dir": stage_ckpt_dir,
                                    "keep_snapshot": True,
                                })
                                _write_heartbeat(workdir, {
                                    "event": "promote",
                                    "step": int(step),
                                    "stage": int(curriculum.stage),
                                    "reason": reason,
                                    "graduated_acc": round(float(frontier_acc), 6),
                                    "last_ckpt_step": int(last_ckpt_step),
                                })

            if config.save_checkpoint and step > 0 and step % config.save_every_steps == 0:
                checkpoints.save_checkpoint_multiprocess(
                    workdir, jax_utils.unreplicate(state), step,
                    keep=ckpt_keep, overwrite=True
                )
                last_ckpt_step = step
                _write_json(os.path.join(workdir, "ckpt_ready.json"), {
                    "event": "periodic",
                    "step": int(step),
                    "stage": int(curriculum.stage),
                    "workdir": workdir,
                    "keep_snapshot": (step % 50000 == 0),
                })
                _write_heartbeat(workdir, {
                    "event": "ckpt",
                    "step": int(step),
                    "stage": int(curriculum.stage),
                    "last_ckpt_step": int(last_ckpt_step),
                })

            if stop_requested["flag"]:
                print(f"[signal] stopping at step {step}", flush=True)
                if config.save_checkpoint:
                    checkpoints.save_checkpoint_multiprocess(
                        workdir, jax_utils.unreplicate(state), step,
                        keep=ckpt_keep, overwrite=True)
                    last_ckpt_step = step
                    _write_json(os.path.join(workdir, "ckpt_ready.json"), {
                        "event": "signal",
                        "step": int(step),
                        "stage": int(curriculum.stage),
                        "workdir": workdir,
                        "keep_snapshot": True,
                    })
                _write_heartbeat(workdir, {
                    "event": "stopped",
                    "step": int(step),
                    "stage": int(curriculum.stage),
                    "last_ckpt_step": int(last_ckpt_step),
                })
                break

        # Final checkpoint at the end of training
        if config.save_checkpoint and not stop_requested["flag"]:
            checkpoints.save_checkpoint_multiprocess(
                workdir, jax_utils.unreplicate(state), config.max_steps,
                keep=ckpt_keep, overwrite=True)
            _write_json(os.path.join(workdir, "ckpt_ready.json"), {
                "event": "final",
                "step": int(config.max_steps),
                "stage": int(curriculum.stage),
                "workdir": workdir,
                "keep_snapshot": True,
            })
        _write_heartbeat(workdir, {
            "event": "done",
            "step": int(last_ckpt_step),
            "stage": int(curriculum.stage),
            "last_ckpt_step": int(last_ckpt_step),
        })