Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+code/elf_l_baseline/checkpoint_64248 filter=lfs diff=lfs merge=lfs -text
+code/elf_l_repa_reg/checkpoint_64248 filter=lfs diff=lfs merge=lfs -text
+gsm8k/elf_b_repa_reg/checkpoint_56352 filter=lfs diff=lfs merge=lfs -text
+math_initialization/elf_l_repa_reg/checkpoint_35220 filter=lfs diff=lfs merge=lfs -text
+math_initialization/elf_l_baseline/checkpoint_35220 filter=lfs diff=lfs merge=lfs -text
+gsm8k/elf_l_repa_reg/checkpoint_56352 filter=lfs diff=lfs merge=lfs -text
+gsm8k/elf_b_baseline/checkpoint_84528 filter=lfs diff=lfs merge=lfs -text
+gsm8k/elf_l_baseline/checkpoint_70440 filter=lfs diff=lfs merge=lfs -text
+math500/elf_l_repa_reg/checkpoint_55824 filter=lfs diff=lfs merge=lfs -text
+math500/elf_l_baseline/checkpoint_55824 filter=lfs diff=lfs merge=lfs -text

code/elf_l_baseline/checkpoint_64248

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d149a8b9f1e8fd020af1294cfad01cb6e4c4b4cec37ffee0a74f6b3e43d7aca
+size 11432492343

code/elf_l_baseline/config.yml

@@ -0,0 +1,113 @@
+# Code: ELF-L baseline
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/train
+eval_data_path:
+- name: mbppplus_test
+  path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/mbppplus_test
+  num_samples: 378
+- name: humanevalplus_test
+  path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/humanevalplus_test
+  num_samples: 164
+max_length: 1024
+max_input_length: 512
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment; REPA stops after epoch 8 of 12
+repa_enabled: false
+
+# REG semantic token
+reg_enabled: false
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 12
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 378
+conditional_eval_metric: evalplus
+training_generation_ema: null
+truncate_generation: false
+online_eval: false
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/code_elf_l_baseline
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: code_elf_l_baseline
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

code/elf_l_repa_reg/checkpoint_64248

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae888e6ab0d6cb5f9eff3d975922d0eaf510fc7756465a3c253adb76eb2592
+size 11646012184

code/elf_l_repa_reg/config.yml

@@ -0,0 +1,136 @@
+# Code: ELF-L REPA+REG
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/train
+eval_data_path:
+- name: mbppplus_test
+  path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/mbppplus_test
+  num_samples: 378
+- name: humanevalplus_test
+  path: data/opencodeinstruct-python-qwen3-embedding-0.6b-input512-max1024-v1/humanevalplus_test
+  num_samples: 164
+max_length: 1024
+max_input_length: 512
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment; REPA stops after epoch 8 of 12
+repa_enabled: true
+repa_depth: 8
+repa_t_min: 0.0
+repa_t_max: 1.0
+repa_projector_dim: 2048
+repa_projector_layers: 3
+repa_projector_type: mlp
+repa_strength: 0.5
+repa_end_fraction: 0.6666666666666666
+repa_align_decoder_rows: true
+repa_loss_type: cos_sim
+repa_teacher_model_name: Qwen/Qwen3-1.7B-Base
+repa_teacher_layer: 20
+repa_teacher_dim: 2048
+repa_teacher_input_format: raw
+repa_prompt_loss_weight: 0.0
+repa_response_loss_weight: 1.0
+repa_shift: 0
+repa_reg_target_source: repa
+
+# REG semantic token
+reg_enabled: true
+reg_projection_topology: shared_text
+reg_loss_weight: 0.1
+reg_teacher_layer: 24
+reg_teacher_dim: 2048
+reg_teacher_pooling: last
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 12
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 378
+conditional_eval_metric: evalplus
+training_generation_ema: null
+truncate_generation: false
+online_eval: false
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/code_elf_l_repa_reg
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: code_elf_l_repa_reg
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

gsm8k/elf_b_baseline/checkpoint_84528

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c60024034fdd18801ad829f14e07a4e485f73bec2b91bb910a971576d68c5b0
+size 3752304931

gsm8k/elf_b_baseline/config.yml

@@ -0,0 +1,113 @@
+# GSM8K: ELF-B baseline
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-B
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: false
+
+# REG semantic token
+reg_enabled: false
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 18
+global_batch_size: 512
+batch_size: 32
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 50
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 1
+eval_freq: 1
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_b_baseline
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_b_baseline
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

gsm8k/elf_b_repa_reg/checkpoint_56352

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2722a1c302a3a566f25385b540d0f7ecb1929fb532166d5f1c003edb7151d237
+size 3955993519

gsm8k/elf_b_repa_reg/config.yml

@@ -0,0 +1,136 @@
+# GSM8K: ELF-B REPA+REG
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-B
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: true
+repa_depth: 4
+repa_t_min: 0.0
+repa_t_max: 1.0
+repa_projector_dim: 2048
+repa_projector_layers: 3
+repa_projector_type: mlp
+repa_strength: 0.5
+repa_end_fraction: 1.0
+repa_align_decoder_rows: true
+repa_loss_type: cos_sim
+repa_teacher_model_name: Qwen/Qwen3-1.7B-Base
+repa_teacher_layer: 20
+repa_teacher_dim: 2048
+repa_teacher_input_format: raw
+repa_prompt_loss_weight: 0.0
+repa_response_loss_weight: 1.0
+repa_shift: 0
+repa_reg_target_source: repa
+
+# REG semantic token
+reg_enabled: true
+reg_projection_topology: shared_text
+reg_loss_weight: 0.1
+reg_teacher_layer: 24
+reg_teacher_dim: 2048
+reg_teacher_pooling: last
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 12
+global_batch_size: 512
+batch_size: 32
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 50
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 1
+eval_freq: 1
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_b_repa_reg
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_b_repa_reg
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

gsm8k/elf_l_baseline/checkpoint_70440

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f62f9d2e0f7624fbd3db058c66a6a0a7890f90ed3c05a605dd1f3cdccb2a98f4
+size 11432492983

gsm8k/elf_l_baseline/config.yml

@@ -0,0 +1,113 @@
+# GSM8K: ELF-L baseline
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: false
+
+# REG semantic token
+reg_enabled: false
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 15
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_l_baseline
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_l_baseline
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

gsm8k/elf_l_repa_reg/checkpoint_56352

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0767a177e9c486b3d8dc359e4842e79a5a4e1109f208e5bfa52976b228c2121b
+size 11646012184

gsm8k/elf_l_repa_reg/config.yml

@@ -0,0 +1,136 @@
+# GSM8K: ELF-L REPA+REG
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: true
+repa_depth: 8
+repa_t_min: 0.0
+repa_t_max: 1.0
+repa_projector_dim: 2048
+repa_projector_layers: 3
+repa_projector_type: mlp
+repa_strength: 0.5
+repa_end_fraction: 1.0
+repa_align_decoder_rows: true
+repa_loss_type: cos_sim
+repa_teacher_model_name: Qwen/Qwen3-1.7B-Base
+repa_teacher_layer: 20
+repa_teacher_dim: 2048
+repa_teacher_input_format: raw
+repa_prompt_loss_weight: 0.0
+repa_response_loss_weight: 1.0
+repa_shift: 0
+repa_reg_target_source: repa
+
+# REG semantic token
+reg_enabled: true
+reg_projection_topology: shared_text
+reg_loss_weight: 0.1
+reg_teacher_layer: 24
+reg_teacher_dim: 2048
+reg_teacher_pooling: last
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 12
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_l_repa_reg
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_l_repa_reg
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

math500/elf_l_baseline/checkpoint_55824

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60189530e7998fa04e417b49b55841b3b475881bec95b760a67647a498b5699
+size 11432492343

math500/elf_l_baseline/config.yml

@@ -0,0 +1,113 @@
+# MATH-500: ELF-L baseline
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/validation_1024
+  num_samples: 1024
+- name: math500_test
+  path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/math500_test
+  num_samples: 500
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: false
+
+# REG semantic token
+reg_enabled: false
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 8
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.25
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 0
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: math_accuracy
+training_generation_ema: 0.999,0.9999
+truncate_generation: false
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization; initialize from GSM8K epoch 7.5, EMA 0.999
+output_dir: outputs/math500_elf_l_baseline
+resume: null
+resume_only_weights: checkpoints/math_initialization/elf_l_baseline/checkpoint_35220
+resume_only_weights_ema: 0.999
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: math500_elf_l_baseline
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

math500/elf_l_repa_reg/checkpoint_55824

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b72ce9be7b59738e8f47826eb5c82b5b1c3ea849d7ab30f401661e221e2e02a3
+size 11646012184

math500/elf_l_repa_reg/config.yml

@@ -0,0 +1,136 @@
+# MATH-500: ELF-L REPA+REG
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/validation_1024
+  num_samples: 1024
+- name: math500_test
+  path: data/openmath-math-qwen3-embedding-0.6b-input256-max1024-5M-v1/math500_test
+  num_samples: 500
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: true
+repa_depth: 8
+repa_t_min: 0.0
+repa_t_max: 1.0
+repa_projector_dim: 2048
+repa_projector_layers: 3
+repa_projector_type: mlp
+repa_strength: 0.5
+repa_end_fraction: 1.0
+repa_align_decoder_rows: true
+repa_loss_type: cos_sim
+repa_teacher_model_name: Qwen/Qwen3-1.7B-Base
+repa_teacher_layer: 20
+repa_teacher_dim: 2048
+repa_teacher_input_format: raw
+repa_prompt_loss_weight: 0.0
+repa_response_loss_weight: 1.0
+repa_shift: 0
+repa_reg_target_source: repa
+
+# REG semantic token
+reg_enabled: true
+reg_projection_topology: shared_text
+reg_loss_weight: 0.1
+reg_teacher_layer: 24
+reg_teacher_dim: 2048
+reg_teacher_pooling: last
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 8
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.25
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 0
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: math_accuracy
+training_generation_ema: 0.999,0.9999
+truncate_generation: false
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization; initialize from GSM8K epoch 7.5, EMA 0.999
+output_dir: outputs/math500_elf_l_repa_reg
+resume: null
+resume_only_weights: checkpoints/math_initialization/elf_l_repa_reg/checkpoint_35220
+resume_only_weights_ema: 0.999
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: math500_elf_l_repa_reg
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

math_initialization/elf_l_baseline/checkpoint_35220

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5785621263f8bb0154192174cd8d4532429831eeede71c73cd0679e90d4dc4c
+size 11432492343

math_initialization/elf_l_baseline/config.yml

@@ -0,0 +1,113 @@
+# GSM8K: ELF-L baseline
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: false
+
+# REG semantic token
+reg_enabled: false
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 15
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_l_baseline
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_l_baseline
+
+# Random seed and data loading
+seed: 42
+num_workers: 8

math_initialization/elf_l_repa_reg/checkpoint_35220

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e11d5dd66aae10f0c0124d622e2d9d2ca981fc10f07f70145d88bfb8293e8c6e
+size 11646012184

math_initialization/elf_l_repa_reg/config.yml

@@ -0,0 +1,136 @@
+# GSM8K: ELF-L REPA+REG
+# Run commands from the repository root.
+
+# Dataset and sequence lengths
+data_path: data/gsm-math-qwen3-embedding-0.6b-v1/train
+eval_data_path:
+- name: validation_1024
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/validation_1024
+  num_samples: 1024
+- name: gsm8k_test
+  path: data/gsm-math-qwen3-embedding-0.6b-v1/gsm8k_test
+  num_samples: 1319
+max_length: 1024
+max_input_length: 256
+pad_token: eos
+use_model_attention_mask: false
+
+# Tokenizer and frozen encoder
+tokenizer_name: Qwen/Qwen3-Embedding-0.6B
+encoder_model_name: Qwen/Qwen3-0.6B-Base
+encoder_dim: 1024
+encoder_layer: 20
+latent_mean: 0.0
+latent_std: 1.0
+
+# ELF architecture
+model: ELF-L
+bottleneck_dim: 128
+num_time_tokens: 4
+num_self_cond_cfg_tokens: 4
+num_model_mode_tokens: 4
+attn_dropout: 0.0
+proj_dropout: 0.0
+
+# REPA auxiliary alignment
+repa_enabled: true
+repa_depth: 8
+repa_t_min: 0.0
+repa_t_max: 1.0
+repa_projector_dim: 2048
+repa_projector_layers: 3
+repa_projector_type: mlp
+repa_strength: 0.5
+repa_end_fraction: 1.0
+repa_align_decoder_rows: true
+repa_loss_type: cos_sim
+repa_teacher_model_name: Qwen/Qwen3-1.7B-Base
+repa_teacher_layer: 20
+repa_teacher_dim: 2048
+repa_teacher_input_format: raw
+repa_prompt_loss_weight: 0.0
+repa_response_loss_weight: 1.0
+repa_shift: 0
+repa_reg_target_source: repa
+
+# REG semantic token
+reg_enabled: true
+reg_projection_topology: shared_text
+reg_loss_weight: 0.1
+reg_teacher_layer: 24
+reg_teacher_dim: 2048
+reg_teacher_pooling: last
+
+# Denoiser objective
+denoiser_p_mean: -1.5
+denoiser_p_std: 0.8
+denoiser_noise_scale: 2.0
+t_eps: 0.05
+time_schedule: logit_normal
+
+# Decoder objective
+decoder_prob: 0.2
+decoder_noise_scale: 1.0
+decoder_p_mean: 0.8
+decoder_p_std: 0.8
+
+# Conditioning
+label_drop_prob: 0.0
+self_cond_prob: 0.5
+self_cond_cfg_min: 0.5
+self_cond_cfg_max: 5.0
+
+# Optimizer and training endpoint
+epochs: 12
+global_batch_size: 512
+batch_size: 16
+blr: 0.001
+lr: 0.002
+lr_schedule: constant
+min_lr: 0.0
+weight_decay: 0.0
+warmup_steps: -1
+warmup_epochs: 0.5
+optimizer: muon
+group_by_length: false
+
+# EMA weights
+ema_decay1:
+- 0.9999
+- 0.999
+ema_warmup_updates: 1000
+
+# Precision and memory
+use_bf16: true
+use_compile: false
+gradient_checkpointing: false
+
+# Training-time evaluation (headline evaluation uses the separate scripts)
+sampling_configs_path: configs/sampling/training.yml
+generation_batch_size: 20
+num_samples: 1024
+conditional_eval_metric: gsm8k_accuracy
+training_generation_ema: null
+truncate_generation: true
+online_eval: true
+
+# Logging and checkpoint frequency (epochs)
+log_freq: 100
+save_freq: 0.5
+eval_freq: 0.5
+
+# Output and initialization
+output_dir: outputs/gsm8k_elf_l_repa_reg
+resume: null
+resume_only_weights: null
+resume_only_weights_ema: null
+
+# Optional experiment tracking
+use_wandb: false
+wandb_project: elf-reg
+wandb_entity: null
+wandb_run_name: gsm8k_elf_l_repa_reg
+
+# Random seed and data loading
+seed: 42
+num_workers: 8