# coding=utf-8 # Copyright 2024 The Google Research Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This file contains function that coordinates the training and evaluation of the model.""" import functools import json import math import os import signal import socket import time import traceback from absl import logging from clu import metric_writers from flax import jax_utils from flax import linen as nn from flax.training import checkpoints import jax from jax import random import jax.numpy as jnp import numpy as np import tensorflow as tf import wandb from train import data from train import evaluater from train import model from train import trainer def log_hyperparams_tb( config, model_config, initial_variables, tf_summary_writer ): """Log hyperparameters to TensorBoard. Args: config: experiment's ConfigDict model_config: model's ConfigDict initial_variables: initial hyperparameter values tf_summary_writer: SummaryWriter object. Returns: The SummaryWriter object and the config. """ # Calculate the total number of model parameters config.num_model_parameters = sum( x.size for x in jax.tree_util.tree_leaves(initial_variables) ) # Convert hyperparameters to tensors config_hyperparameters = [ tf.convert_to_tensor([k, str(v)]) for k, v in config.items() ] model_config_hyperparameters = [ tf.convert_to_tensor([k, str(v)]) for k, v in model_config.__dict__.items() ] # Log model hyperparameters to TensorBoard with tf_summary_writer.as_default(): tf.summary.text( "Model hyperparameters", tf.stack(model_config_hyperparameters), step=0 ) tf.summary.text( "Config hyperparameters", tf.stack(config_hyperparameters), step=0 ) return tf_summary_writer, config def _write_json(path, payload): """Atomic JSON write so a sidecar never reads a half-written file.""" tmp = path + ".tmp" with open(tmp, "w") as f: json.dump(payload, f, indent=2, sort_keys=True) f.write("\n") os.replace(tmp, path) def _write_heartbeat(workdir, payload): payload = dict(payload) payload.setdefault("host", socket.gethostname()) payload.setdefault("job_id", os.environ.get("SLURM_JOB_ID", "")) payload["unix_time"] = time.time() payload["iso_time"] = time.strftime("%Y-%m-%dT%H:%M:%S%z") slim = {k: payload[k] for k in ("event", "step", "stage", "loss", # val_acc = P(digit not in the stage-t candidate set) "val_acc", "puzzle_acc", # location order, tolerant to ties within a propagation wave "loc_wave", "loc_coverage", "loc_lcs", "loc_dup", "loc_acc", # superposition: excess 0 == uniform over the candidate set "val_excess", "val_mass", "val_spread", "val_excess_multi", "val_mass_multi", "val_out_multi", "val_kl_multi", "last_ckpt_step", "error") if k in payload} _write_json(os.path.join(workdir, "heartbeat.json"), slim) def train_and_evaluate(config, workdir): """The training and evaluation loops for the model. Args: config: experiment's config dictionary. workdir: directory to use for logging. """ # Orbax checkpointing requires an absolute path. workdir = os.path.abspath(workdir) logging.info("Creating training and evaluator dataset iterator") curriculum = data.CurriculumState( stage=int(getattr(config, "curriculum_start_stage", 1)), max_stage=int(getattr(config, "curriculum_max_stage", 6))) train_data_iter = data.create_iter( config, config.minibatch_size, train=True, curriculum=curriculum) eval_data_iter = data.create_iter(config, config.minibatch_size, train=False) logging.info("Finished creating training dataset iterator") model_config = model.TransformerConfig( dtype=config.dtype, vocab_size=config.vocab_size, seq_len=config.seq_len, num_heads=config.num_heads, num_layers=config.num_layers, emb_dim=config.emb_dim, qkv_dim=config.qkv_dim, mlp_dim=config.mlp_dim, dropout_rate=config.dropout_rate, attention_dropout_rate=config.attention_dropout_rate, deterministic=False, num_latent_slots=int(config.num_latent_slots), inject_latents=bool(int(getattr(config, "recurrent_latent", 1))), ) logging.info("train_config: %s", str(model_config.__dict__)) print(str(model_config.__dict__), flush=True) rng = jax.random.PRNGKey(config.seed) rng, init_rng, inference_rng = random.split(rng, num=3) # Initialize the model and get initial variables. Dummy latent arguments # are provided so the latent projector parameters are created at init. rng, dropout_rng = jax.random.split(rng) input_shape = (config.minibatch_size, config.seq_len) net = model.TransformerLMHeadModel(model_config) rng_keys = {"params": init_rng, "dropout": dropout_rng} K = int(config.num_latent_slots) dummy_latents = jnp.zeros( (config.minibatch_size, K, config.emb_dim), model_config.dtype) dummy_positions = jnp.zeros((config.minibatch_size, K), jnp.int32) dummy_active = jnp.zeros((config.minibatch_size, K), bool) sample_out, initial_variables = jax.jit( net.init_with_output )(rng_keys, jnp.ones(input_shape, jnp.int32), dummy_latents, dummy_positions, dummy_active) state, lr_scheduler_fn = trainer.get_state(config, net, initial_variables) # Resume-and-extend support: when resuming, start the training loop at the # restored optimizer step (not 0) so a larger config.max_steps continues the # cosine LR schedule cleanly instead of re-running steps or overshooting. start_step = 0 if config.resume_training: state = checkpoints.restore_checkpoint(config.ckpt_loc, state) start_step = int(state.step) print("----------Restored model from", config.ckpt_loc, f"at step {start_step}-----------") writer = metric_writers.create_default_writer( workdir, asynchronous=False, just_logging=(jax.process_index() > 0)) tf_summary_writer = tf.summary.create_file_writer(workdir) logging.info("config: %s", str(config.__dict__)) state = jax_utils.replicate(state) dropout_rngs = jax.random.split(rng, jax.local_device_count()) def make_p_train_step(num_passes): return jax.pmap( functools.partial( trainer.train_step, config=model_config, hyperparams=config, learning_rate_fn=lr_scheduler_fn, num_passes=num_passes), axis_name="batch", donate_argnums=(0,)) # num_passes = latent recurrence depth for this stage, capped by the number # of latent slots (K=0 -> 0 passes -> no-latent control baseline). # When recurrence is disabled (control), force 0 passes so no hidden state # is ever fed back: the slots become static, independent per-stage readouts. recurrent = bool(int(getattr(config, "recurrent_latent", 1))) passes_per_stage = int(getattr(config, "passes_per_stage", 1)) def passes_for(stage): return min(passes_per_stage * stage, K) if recurrent else 0 p_train_step = make_p_train_step(passes_for(curriculum.stage)) p_eval_step = jax.pmap(functools.partial(evaluater.eval_step, config=model_config.replace(deterministic=True)), axis_name="batch") hooks, report_progress, train_metrics = trainer.get_metrics_report_progress( config, workdir, writer) tf_summary_writer, config = log_hyperparams_tb( config, model_config, initial_variables, tf_summary_writer ) # Instance-arm promotion is THREE gates, all required, no patience/plateau: # val_mass_multi >= promote_mass_threshold (support inside S) # val_spread >= promote_spread_threshold (uniform over S) # val_hbound >= promote_hbound_threshold (H(p) <= log|S| per cell) # S is the raw per-cell candidate set from the wave mask, not the filtered # instance support after constraint solving. promote_mass_threshold = float(getattr(config, "promote_acc_threshold", 0.85)) promote_spread_threshold = float( getattr(config, "promote_spread_threshold", 0.85)) promote_hbound_threshold = float( getattr(config, "promote_hbound_threshold", 0.0)) # When set, this is the ONLY value gate: CE(q||p) <= log|S| already implies # both "mass is inside S" and "no candidate was dropped", so mass*/spread*/ # hbound become diagnostics. See _inst_ce_stats in evaluater.py. promote_ceq_threshold = float(getattr(config, "promote_ceq_threshold", 0.0)) # Applies to loc_wave (order up to wave ties), not the positional loc_acc. # 0 disables the gate. promote_loc_threshold = float(getattr(config, "promote_loc_threshold", 0.0)) promote_patience = int(getattr(config, "promote_patience_steps", 8000)) min_stage_steps = int(getattr(config, "min_stage_steps", 2000)) plateau_steps = int(getattr(config, "plateau_steps", 0)) plateau_delta = float(getattr(config, "plateau_delta", 0.005)) instance_mode = bool(getattr(config, "instance_dir", None)) per_stage_inset = {} per_stage_vexcess, per_stage_vmass, per_stage_vspread = {}, {}, {} per_stage_vexc_m, per_stage_vmass_m, per_stage_vkl_m = {}, {}, {} per_stage_vhb, per_stage_vhgap, per_stage_vhb_puz = {}, {}, {} per_stage_ceq, per_stage_hq, per_stage_ceq_ok = {}, {}, {} per_stage_ceq_gap = {} # Best frontier-depth accuracy seen in the current stage, and the step it # was last improved: the plateau detector's state. stage_best_acc = -1.0 stage_best_step = start_step ckpt_keep = int(getattr(config, "ckpt_keep", 100)) # Protected directory for per-stage checkpoints (never rolled off). stage_ckpt_dir = os.path.join(workdir, "stage_ckpts") stage_started_at = start_step last_ckpt_step = start_step stop_requested = {"flag": False} def _on_stop(signum, _frame): print(f"[signal] {signum} received; will save and exit after this step", flush=True) stop_requested["flag"] = True signal.signal(signal.SIGTERM, _on_stop) signal.signal(signal.SIGINT, _on_stop) _write_heartbeat(workdir, { "event": "start", "step": start_step, "stage": curriculum.stage, "max_steps": int(config.max_steps), }) with metric_writers.ensure_flushes(writer): for step in range(start_step, config.max_steps): if step%10000 == 0: print("Step:", step, flush=True) state, metrics = trainer.train_one_step(p_train_step, config, state, step, dropout_rngs, train_data_iter) for h in hooks: h(step) if math.isnan(metrics["loss"][0]): print("The loss function became nan: This might be due to the choice of hyperparameters.") _write_heartbeat(workdir, { "event": "nan_loss", "step": step, "stage": curriculum.stage, }) break if step % config.eval_every_steps == 0: try: eval_metrics = evaluater.get_eval_metrics( state, eval_data_iter, p_eval_step, config) except Exception: traceback.print_exc() _write_heartbeat(workdir, { "event": "eval_error", "step": step, "stage": curriculum.stage, "loss": float(metrics["loss"].mean()), "ce": float(metrics["ce_loss"].mean()), "error": traceback.format_exc()[-2000:], }) print("[eval] failed; skipping this eval and continuing", flush=True) eval_metrics = None if eval_metrics is not None: per_level = eval_metrics.pop("per_level_acc") per_slot = eval_metrics.pop("per_slot_acc", {}) per_slot_ch = eval_metrics.pop("per_slot_acc_changed", {}) per_stage_inset = eval_metrics.pop("per_stage_inset_acc", {}) per_stage_vexcess = eval_metrics.pop( "per_stage_val_excess", {}) per_stage_vmass = eval_metrics.pop("per_stage_val_mass", {}) per_stage_vspread = eval_metrics.pop( "per_stage_val_spread", {}) per_stage_vexc_m = eval_metrics.pop( "per_stage_val_excess_multi", {}) per_stage_vmass_m = eval_metrics.pop( "per_stage_val_mass_multi", {}) per_stage_vkl_m = eval_metrics.pop( "per_stage_val_kl_multi", {}) per_stage_vhb = eval_metrics.pop( "per_stage_val_hbound", {}) per_stage_vhgap = eval_metrics.pop( "per_stage_val_hgap", {}) per_stage_vhb_puz = eval_metrics.pop( "per_stage_val_hbound_puz", {}) per_stage_ceq = eval_metrics.pop("per_stage_val_ceq", {}) per_stage_hq = eval_metrics.pop("per_stage_val_hq", {}) per_stage_ceq_ok = eval_metrics.pop( "per_stage_val_ceq_ok", {}) per_stage_ceq_gap = eval_metrics.pop( "per_stage_val_ceq_gap", {}) eval_metrics.pop("per_stage_val_ceq_puz", {}) eval_metrics.pop("per_stage_val_ce", {}) eval_metrics.pop("per_stage_val_floor", {}) per_bin = eval_metrics.pop("per_bin_acc", {}) def _m(key): v = eval_metrics.get(key, []) return round(float(np.mean(v)), 4) if len(v) else -1.0 def _s(d): return round(float(d.get(curriculum.stage, -1.0)), 4) # val_acc = P(digit not in stage-t candidate set), # teacher-forced so location cannot contaminate it. # 0 = all mass inside S; chance at stage 0 is ~0.59. mass = _s(per_stage_vmass) val_out = round(1.0 - mass, 4) if mass >= 0 else -1.0 mass_m = _s(per_stage_vmass_m) val_out_m = round(1.0 - mass_m, 4) if mass_m >= 0 else -1.0 print(step, "stage", curriculum.stage, "loss", round(float(metrics["loss"].mean()), 4), "mass*", mass_m, "spread*", _s(per_stage_vspread), "hbound", _s(per_stage_vhb), # ceq must sit under log|S| and above H(q); ceq_ok is # the fraction of cells that manage it. "ceq", _s(per_stage_ceq), "H(q)", _s(per_stage_hq), "ceq_ok", _s(per_stage_ceq_ok), "val_acc", val_out_m, flush=True) _write_heartbeat(workdir, { "event": "eval", "step": int(step), "stage": int(curriculum.stage), "loss": round(float(metrics["loss"].mean()), 6), "ce": round(float(metrics["ce_loss"].mean()), 6), "val_acc": val_out_m, "puzzle_acc": _m("acc_complete_puzzle"), # Permutation-tolerant location signals; loc_acc is the # old positional match, kept only for continuity. "loc_wave": _m("loc_wave"), "loc_coverage": _m("loc_coverage"), "loc_lcs": _m("loc_lcs"), "loc_dup": _m("loc_dup"), "loc_acc": _m("loc_acc"), # Superposition: 0 excess == uniform over the stage's # candidate set; spread 1.0 == no mode collapse. "val_excess": _s(per_stage_vexcess), "val_mass": _s(per_stage_vmass), "val_spread": _s(per_stage_vspread), # |S|>=2 cells only: excess_multi == log(1/mass_multi) # + kl_multi, so it is the single number that falls only # when leakage and non-uniformity both fall. "val_excess_multi": _s(per_stage_vexc_m), "val_mass_multi": mass_m, "val_out_multi": val_out_m, "val_kl_multi": _s(per_stage_vkl_m), # Entropy ceiling H(p) <= log|S|: fraction of |S|>=2 # cells that satisfy it, the mean margin in nats, and # the stricter all-cells-in-the-puzzle view. "val_hbound": _s(per_stage_vhb), "val_hgap": _s(per_stage_vhgap), "val_hbound_puz": _s(per_stage_vhb_puz), # The promotion criterion and the two numbers that # bracket it: H(q) <= ceq should hold, ceq <= log|S| is # the gate, and ceq_gap is the signed slack. "val_ceq": _s(per_stage_ceq), "val_hq": _s(per_stage_hq), "val_ceq_ok": _s(per_stage_ceq_ok), "val_ceq_gap": _s(per_stage_ceq_gap), "last_ckpt_step": int(last_ckpt_step), }) with tf_summary_writer.as_default(): tf.summary.scalar("loss", metrics["loss"].mean(), step=step) tf.summary.scalar("ce_loss", metrics["ce_loss"].mean(), step=step) tf.summary.scalar("aux_bce_loss", metrics["aux_loss"].mean(), step=step) tf.summary.scalar( "learning rate", metrics["learning_rate"].mean(), step=step ) tf.summary.scalar("curriculum_stage", curriculum.stage, step=step) log_dict = {'loss': metrics["loss"].mean(), 'learning rate': metrics["learning_rate"].mean()} for key in eval_metrics.keys(): vals = eval_metrics[key] if not vals: continue tf.summary.scalar( "eval_" + key, np.array(vals).mean(), step=step ) log_dict[ "eval_" + key ] = np.array(vals).mean() for lvl, v in per_level.items(): if v >= 0: tf.summary.scalar(f"eval_acc_level_{lvl}", v, step=step) log_dict[f"eval_acc_level_{lvl}"] = v for s, v in per_slot.items(): if v >= 0: tf.summary.scalar(f"eval_cand_depth_{s}", v, step=step) log_dict[f"eval_cand_depth_{s}"] = v for s, v in per_slot_ch.items(): if v >= 0: tf.summary.scalar(f"eval_cand_depth_chg_{s}", v, step=step) if config.use_wandb: wandb.log(log_dict, step=step) # ---- Curriculum promotion ---- # Instance arm: THREE gates, all required, no plateau, no # patience. The model must actually learn the current stage's # representation before the next wave is introduced. # 1. val_mass_multi >= mass_bar: support is inside S # 2. val_spread >= spread_bar: support is uniform on S # 3. val_hbound >= hbound_bar: H(p) <= log|S| on that # fraction of cells. Checked per CELL and then counted, # because the inequality is per cell: comparing mean # H(p) to mean log|S| would let comfortable cells mask # violating ones. Catches diffuse leak that mass* is # blind to, since spreading a fixed leak over many # illegal digits costs mass the same but entropy more. # S is the raw wave-solver candidate set at that cell, NOT # the support of the filtered instances after constraints. rounds_curric = str( getattr(config, "data_curriculum", "none")) == "rounds" has_frontier = int(config.num_latent_slots) > 0 or rounds_curric if has_frontier and curriculum.stage < curriculum.max_stage: t = curriculum.stage loc_now = _m("loc_wave") mass_m = float(per_stage_vmass_m.get(t, -1.0)) spread_now = float(per_stage_vspread.get(t, -1.0)) hbound_now = float(per_stage_vhb.get(t, -1.0)) if instance_mode: frontier_acc = mass_m elif int(config.num_latent_slots) > 0: frontier_acc = per_slot_ch.get(t, -1.0) if frontier_acc < 0: frontier_acc = per_slot.get(t, -1.0) else: frontier_acc = per_bin.get(t, -1.0) if frontier_acc < 0: seen = [v for b, v in per_bin.items() if b <= t and v >= 0] frontier_acc = (float(np.mean(seen)) if seen else -1.0) steps_in_stage = step - stage_started_at measured = frontier_acc >= 0 if measured and frontier_acc > stage_best_acc + plateau_delta: stage_best_acc = frontier_acc stage_best_step = step loc_ready = (promote_loc_threshold <= 0.0 or loc_now >= promote_loc_threshold) ceq_ok_now = float(per_stage_ceq_ok.get(t, -1.0)) if instance_mode and promote_ceq_threshold > 0.0: # Single criterion. CE(q||p) <= log|S| is violated by # leak and by collapse alike, so the mass and spread # bars would only add mutually infeasible constraints # on top of it (see probe_entropy_ceiling.py). hit_threshold = (ceq_ok_now >= 0 and loc_ready and ceq_ok_now >= promote_ceq_threshold) stalled = False patience_over = False elif instance_mode: mass_ready = mass_m >= promote_mass_threshold spread_ready = spread_now >= promote_spread_threshold hbound_ready = (promote_hbound_threshold <= 0.0 or hbound_now >= promote_hbound_threshold) hit_threshold = (measured and loc_ready and mass_ready and spread_ready and hbound_ready) # No plateau, no patience: stay on this stage until # both superposition gates clear. stalled = False patience_over = False else: hit_threshold = (measured and loc_ready and frontier_acc >= promote_mass_threshold) stalled = (measured and loc_ready and plateau_steps > 0 and (step - stage_best_step) >= plateau_steps) patience_over = (promote_patience > 0 and steps_in_stage >= promote_patience) if steps_in_stage >= min_stage_steps and ( hit_threshold or stalled or patience_over): reason = ("threshold" if hit_threshold else "plateau" if stalled else "patience") curriculum.stage += 1 stage_started_at = step stage_best_acc = -1.0 stage_best_step = step p_train_step = make_p_train_step(passes_for(curriculum.stage)) what = ("round-bin" if int(config.num_latent_slots) == 0 else "depth") if instance_mode and promote_ceq_threshold > 0.0: extra = (f" ceq_ok={ceq_ok_now:.3f}" f" ceq={float(per_stage_ceq.get(t, -1)):.3f}" f" H(q)={float(per_stage_hq.get(t, -1)):.3f}") elif instance_mode: extra = (f" mass*={mass_m:.3f}" f" spread*={spread_now:.3f}" f" hbound={hbound_now:.3f}") else: extra = f" acc={frontier_acc:.3f}" print(f"[curriculum] step {step}: promote to stage " f"{curriculum.stage} ({reason}; graduated {what} " f"{t}{extra} after {steps_in_stage} steps); " f"latent passes={curriculum.stage}, " f"pool/snapshots now 1..{curriculum.stage}", flush=True) if config.save_checkpoint: unrep_state = jax_utils.unreplicate(state) # Rolling checkpoint in the main workdir. checkpoints.save_checkpoint_multiprocess( workdir, unrep_state, step, keep=ckpt_keep, overwrite=True) # Protected copy: the model *entering* this stage, # kept permanently under stage_ckpts/ (never rolled # off), so every stage's checkpoint survives. checkpoints.save_checkpoint_multiprocess( stage_ckpt_dir, unrep_state, step, keep=100, overwrite=True, prefix=f"stage{curriculum.stage}_") last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "stage", "step": int(step), "stage": int(curriculum.stage), "reason": reason, "workdir": workdir, "stage_ckpt_dir": stage_ckpt_dir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "promote", "step": int(step), "stage": int(curriculum.stage), "reason": reason, "graduated_acc": round(float(frontier_acc), 6), "last_ckpt_step": int(last_ckpt_step), }) if config.save_checkpoint and step > 0 and step % config.save_every_steps == 0: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), step, keep=ckpt_keep, overwrite=True ) last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "periodic", "step": int(step), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": (step % 50000 == 0), }) _write_heartbeat(workdir, { "event": "ckpt", "step": int(step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), }) if stop_requested["flag"]: print(f"[signal] stopping at step {step}", flush=True) if config.save_checkpoint: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), step, keep=ckpt_keep, overwrite=True) last_ckpt_step = step _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "signal", "step": int(step), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "stopped", "step": int(step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), }) break # Final checkpoint at the end of training if config.save_checkpoint and not stop_requested["flag"]: checkpoints.save_checkpoint_multiprocess( workdir, jax_utils.unreplicate(state), config.max_steps, keep=ckpt_keep, overwrite=True) _write_json(os.path.join(workdir, "ckpt_ready.json"), { "event": "final", "step": int(config.max_steps), "stage": int(curriculum.stage), "workdir": workdir, "keep_snapshot": True, }) _write_heartbeat(workdir, { "event": "done", "step": int(last_ckpt_step), "stage": int(curriculum.stage), "last_ckpt_step": int(last_ckpt_step), })