Add all seven checkpoints for completed R3 A0-R1, A2-R1 and multilayer FBW arms

Native safetensors and pinned configurations. Exact source tensor equality, finite tensors and strict native loading verified on CPU. Existing files and repository visibility preserved.

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 12000
+
+Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_12000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 12000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 1536000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 12000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 3145728000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 12000,
+  "files": {
+    "README.md": {
+      "sha256": "ecfcad9dcd20100fe9c87281508367495a2e47f6ec745b20fad8455b5879fd1f",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "6e6a9ced87ea8f55c9ab0533b901e84e97991d5aa0838f0b29ee213e9639793a",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_12000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_12000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 16000
+
+Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_16000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 16000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 2048000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 16000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 4194304000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 16000,
+  "files": {
+    "README.md": {
+      "sha256": "d3f4a9de16bf052561064000ad77015e88319f11afa1b6c0d01d3628ec9dc4b6",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "508f1517705df16a8bd79f1b0c9ed5c610679485964d5c670e95fbafa6f83d9a",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_16000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_16000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 20000
+
+Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_20000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 20000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 2560000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 20000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 5242880000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 20000,
+  "files": {
+    "README.md": {
+      "sha256": "e5c48096d83b5fbb9c8ba691b37f3f483a5cf1c67ef37d9dd0bc90c072fe1b18",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "0b26f395c2623ab007efcafec34a04fd0b3238d47d690b1bc2cf2cb9d2116737",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_20000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_20000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 24000
+
+Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_24000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 24000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 3072000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 24000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 6291456000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 24000,
+  "files": {
+    "README.md": {
+      "sha256": "a2d3faf19e2eea6312eb85940944b0afbe84dfcc25d302616f2c66549bfe8ad2",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "5619a51676440252ea6ff2c78de18c37322142cacc04d0188457faa0e28c4d8d",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_24000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_24000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 26703
+
+Final checkpoint at step **26,703**, after **7,000,031,232 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_26703")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 26703,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 3417984,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": true,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 26703,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 7000031232,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 26703,
+  "files": {
+    "README.md": {
+      "sha256": "d5a7044737b80db05eaf39029237d2df47cfd6135677146decb1997732ae5266",
+      "size": 2692
+    },
+    "config.json": {
+      "sha256": "0be3be4a6e3da7092a2bc29821ced20389b0298ede9879790c9762c65ff47ea3",
+      "size": 12272
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_26703",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_26703/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 4000
+
+Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_4000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 4000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 512000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 4000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 1048576000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 4000,
+  "files": {
+    "README.md": {
+      "sha256": "754c86bf86065e0f121d51b1f3046858ead7aaea160a46caa6db1ddc72cb127d",
+      "size": 2696
+    },
+    "config.json": {
+      "sha256": "880053416c618d5a37cb0de24ba8dfdd88418832d970f89d2e55311474136a81",
+      "size": 12270
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_4000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_4000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/README.md

@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 8000
+
+Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_8000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/config.json

@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 8000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 1024000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 8000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 2097152000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/launch_config.env

@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 8000,
+  "files": {
+    "README.md": {
+      "sha256": "aab5de6678fc9e11bc6f70fee55ae08a3f7dd392815042ddf025b0f3ba6b41a7",
+      "size": 2696
+    },
+    "config.json": {
+      "sha256": "5ef6a6b687625bc4d8ebabb0eef5da14e72a9133cc48317b27fe086fb945cee6",
+      "size": 12271
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_8000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5
+size 1152254688

fbw_L2L11L20_7b_seed42_20260920-023045/step_8000/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/README.md

@@ -0,0 +1,37 @@
+# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 12000
+
+Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed.
+
+Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_12000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**config["model_config"]),
+                                 LookaheadConfig(**config["lookahead_config"]))
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/config.json

@@ -0,0 +1,205 @@
+{
+  "activation": "squared_relu",
+  "activation_checkpointing": false,
+  "allow_objective_change_on_resume": false,
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "block_application_multiplier": 6.8999999999999995,
+  "cache_layer_indices": [
+    0,
+    10,
+    19
+  ],
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 12000,
+  "checkpoint_refiner": false,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 1536000,
+  "crop_padding_before_loss": true,
+  "draft_loss_weight": 0.0,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false,
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CACHE_LAYERS": null,
+    "LOOKAHEAD_CHECKPOINT_REFINER": false,
+    "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0,
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0,
+    "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6,
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0,
+    "LOOKAHEAD_GRADIENT_CHECKPOINTING": false,
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": 3,
+    "LOOKAHEAD_PSEUDO_INITIALIZER": "A0",
+    "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0,
+    "LOOKAHEAD_PSEUDO_TOP_K": 128,
+    "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0,
+    "LOOKAHEAD_REFINER_FFN_SIZE": 5632,
+    "LOOKAHEAD_REFINER_GATE_BIAS": -2.0,
+    "LOOKAHEAD_REFINER_VARIANT": "R1",
+    "LOOKAHEAD_SLIDING_WINDOW": 2048,
+    "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919",
+    "WANDB_RUN_ID": "rsk51lmo",
+    "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1"
+  },
+  "execution_architecture": "lookahead",
+  "feedback_loss_weight": 1.0,
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "future_gradient_mode": "all",
+  "future_loss_weight": 0.0,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refined_future_loss_weight": 1.0,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R1"
+  },
+  "lookahead_mode": "parallel",
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "non_tensor_state_dict_entries": {},
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "lookahead_parallel_final_pair",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "backbone_total_trainable": 566144128,
+    "feedback_parameters": 3964928,
+    "lookahead_parameters": 101114112,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_parameters": 667258240,
+    "total_trainable": 667258240
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "pseudo_backend": "sdpa_auto",
+  "pseudo_initializer": "A0",
+  "pseudo_temperature": 1.0,
+  "pseudo_top_k": 128,
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "refined_future_loss_weight": 1.0,
+  "refiner_activation": "silu_gated",
+  "refiner_variant": "R1",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 195,
+  "tensor_dtypes": {
+    "torch.bfloat16": 146,
+    "torch.float32": 49
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 667258240,
+  "total_trainable_parameters": 667258240,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1",
+  "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+  "training_tokens": 3145728000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/launch_config.env

@@ -0,0 +1,12 @@
+# Supported design cell, not a scheduled experiment; see ablation-framework.md.
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=lookahead_d0-a0-r1
+# Final-slot objective; no direct CE on either pre-refiner latent.
+LOOKAHEAD_DRAFT_LOSS_WEIGHT=0
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0
+LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1
+LOOKAHEAD_PSEUDO_INITIALIZER=A0
+LOOKAHEAD_REFINER_VARIANT=R1
+# Keep the refiner FFN width identical across R0 and R1.
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+WANDB_TAGS=product,lookahead,ablation,d0-a0-r1

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 12000,
+  "files": {
+    "README.md": {
+      "sha256": "288c505d627776788e3bb01589c9c0736b8e6e56365af3d40b31f0f75a3e93e6",
+      "size": 2779
+    },
+    "config.json": {
+      "sha256": "2fb5cfd1d138c8f68feef4cc242b86a61d5f273cd218b891279209cb0a465a87",
+      "size": 7332
+    },
+    "launch_config.env": {
+      "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c",
+      "size": 523
+    },
+    "model.safetensors": {
+      "sha256": "2e0ecf712296d92827e8e30f46a8ea3af16fec22215d5f364e9b09131c09442a",
+      "size": 1334676968
+    },
+    "training_recipe.json": {
+      "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329",
+      "size": 4393
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651"
+}

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e0ecf712296d92827e8e30f46a8ea3af16fec22215d5f364e9b09131c09442a
+size 1334676968

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000/training_recipe.json

@@ -0,0 +1,151 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 667258240,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 101114112,
+      "total_parameters": 667258240
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_final_pair",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.8999999999999995,
+    "future_loss_weight": 0.0,
+    "future_gradient_mode": "all",
+    "draft_loss_weight": 0.0,
+    "refined_future_loss_weight": 1.0,
+    "pseudo_initializer": "A0",
+    "pseudo_top_k": 128,
+    "pseudo_temperature": 1.0,
+    "refiner_variant": "R1",
+    "allow_objective_change_on_resume": false
+  }
+}

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_16000/README.md

@@ -0,0 +1,37 @@
+# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 16000
+
+Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed.
+
+Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_16000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**config["model_config"]),
+                                 LookaheadConfig(**config["lookahead_config"]))
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.

r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_16000/config.json

@@ -0,0 +1,205 @@
+{
+  "activation": "squared_relu",
+  "activation_checkpointing": false,
+  "allow_objective_change_on_resume": false,
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "block_application_multiplier": 6.8999999999999995,
+  "cache_layer_indices": [
+    0,
+    10,
+    19
+  ],
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 16000,
+  "checkpoint_refiner": false,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 2048000,
+  "crop_padding_before_loss": true,
+  "draft_loss_weight": 0.0,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false,
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CACHE_LAYERS": null,
+    "LOOKAHEAD_CHECKPOINT_REFINER": false,
+    "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0,
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0,
+    "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6,
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0,
+    "LOOKAHEAD_GRADIENT_CHECKPOINTING": false,
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": 3,
+    "LOOKAHEAD_PSEUDO_INITIALIZER": "A0",
+    "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0,
+    "LOOKAHEAD_PSEUDO_TOP_K": 128,
+    "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0,
+    "LOOKAHEAD_REFINER_FFN_SIZE": 5632,
+    "LOOKAHEAD_REFINER_GATE_BIAS": -2.0,
+    "LOOKAHEAD_REFINER_VARIANT": "R1",
+    "LOOKAHEAD_SLIDING_WINDOW": 2048,
+    "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919",
+    "WANDB_RUN_ID": "rsk51lmo",
+    "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1"
+  },
+  "execution_architecture": "lookahead",
+  "feedback_loss_weight": 1.0,
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "future_gradient_mode": "all",
+  "future_loss_weight": 0.0,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refined_future_loss_weight": 1.0,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R1"
+  },
+  "lookahead_mode": "parallel",
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "non_tensor_state_dict_entries": {},
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "lookahead_parallel_final_pair",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "backbone_total_trainable": 566144128,
+    "feedback_parameters": 3964928,
+    "lookahead_parameters": 101114112,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_parameters": 667258240,
+    "total_trainable": 667258240
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "pseudo_backend": "sdpa_auto",
+  "pseudo_initializer": "A0",
+  "pseudo_temperature": 1.0,
+  "pseudo_top_k": 128,
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "refined_future_loss_weight": 1.0,
+  "refiner_activation": "silu_gated",
+  "refiner_variant": "R1",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 195,
+  "tensor_dtypes": {
+    "torch.bfloat16": 146,
+    "torch.float32": 49
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 667258240,
+  "total_trainable_parameters": 667258240,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1",
+  "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+  "training_tokens": 4194304000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}