Add files using upload-large-folder tool

outputs/temporal_nope/a0_compact_artifact_smoke_seed0_kerrigan/resolved_config.yaml

@@ -0,0 +1,133 @@
+independent_first_frame: false
+warp_denoising_step: true
+weight_decay: 0.01
+same_step_across_blocks: true
+discriminator_lr_multiplier: 1.0
+last_step_only: false
+i2v: false
+num_training_frames: 21
+gc_interval: 100
+context_noise: 0
+causal: true
+ckpt_step: 0
+prompt_name: MovieGenVideoBench
+prompt_path: prompts/MovieGenVideoBench.txt
+eval_first_n: 64
+num_samples: 1
+height: 480
+width: 832
+num_frames: 81
+experiment:
+  id: a0_compact_artifact_smoke_seed0_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: Original RoPE
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 1.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+training:
+  enabled: false
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 2000
+  checkpoint_steps:
+  - 0
+  - 100
+  - 250
+  - 500
+  - 1000
+  - 2000
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 2.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a0_compact_artifact_smoke_seed0_kerrigan_rerun1/resolved_config.yaml

@@ -0,0 +1,133 @@
+independent_first_frame: false
+warp_denoising_step: true
+weight_decay: 0.01
+same_step_across_blocks: true
+discriminator_lr_multiplier: 1.0
+last_step_only: false
+i2v: false
+num_training_frames: 21
+gc_interval: 100
+context_noise: 0
+causal: true
+ckpt_step: 0
+prompt_name: MovieGenVideoBench
+prompt_path: prompts/MovieGenVideoBench.txt
+eval_first_n: 64
+num_samples: 1
+height: 480
+width: 832
+num_frames: 81
+experiment:
+  id: a0_compact_artifact_smoke_seed0_kerrigan_rerun1
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: Original RoPE
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 1.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+training:
+  enabled: false
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 2000
+  checkpoint_steps:
+  - 0
+  - 100
+  - 250
+  - 500
+  - 1000
+  - 2000
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 2.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/checkpoint_model_000100/model.pt", "sha256": "1a2cdd3720e9823e69bc3542e83e0758e3e5ce235fd1d02fd12612259f04bb5f", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/checkpoint_model_000200/model.pt", "sha256": "026104aeeaa41d4df33f2d6289a002e734f286ac4ed160ccd7bcb33dbb906b57", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/resolved_config.yaml

@@ -0,0 +1,118 @@
+experiment:
+  id: a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: RoPE-CPT matched control
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 1.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 1.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/train.log

@@ -0,0 +1,259 @@
+W0805 22:45:23.818000 2683199 site-packages/torch/distributed/run.py:793] 
+W0805 22:45:23.818000 2683199 site-packages/torch/distributed/run.py:793] *****************************************
+W0805 22:45:23.818000 2683199 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0805 22:45:23.818000 2683199 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank7]:[W805 22:45:34.911466922 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank2]:[W805 22:45:34.956285476 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank4]:[W805 22:45:34.086420255 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W805 22:45:34.106483647 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank1]:[W805 22:45:34.130572991 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W805 22:45:34.133215750 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank6]:[W805 22:45:34.135853058 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank0]:[W805 22:45:34.477965848 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.06264185 alpha=1.000000 samples=32 seconds=24.102
+step=1 loss=0.16898617 grad_norm=0.284459 alpha=1.000000 seconds=33.804
+step=2 loss=0.08548040 grad_norm=0.263723 alpha=1.000000 seconds=33.385
+step=3 loss=0.07998533 grad_norm=0.321230 alpha=1.000000 seconds=33.640
+step=4 loss=0.11531127 grad_norm=0.292704 alpha=1.000000 seconds=33.582
+step=5 loss=0.04095566 grad_norm=0.441111 alpha=1.000000 seconds=33.764
+step=6 loss=0.05031534 grad_norm=0.246053 alpha=1.000000 seconds=33.774
+step=7 loss=0.12238266 grad_norm=0.300544 alpha=1.000000 seconds=33.602
+step=8 loss=0.04164436 grad_norm=0.280257 alpha=1.000000 seconds=33.747
+step=9 loss=0.04078319 grad_norm=0.290209 alpha=1.000000 seconds=33.664
+step=10 loss=0.11827542 grad_norm=0.361131 alpha=1.000000 seconds=33.763
+step=11 loss=0.05484041 grad_norm=0.295932 alpha=1.000000 seconds=33.694
+step=12 loss=0.04062699 grad_norm=0.225317 alpha=1.000000 seconds=33.644
+step=13 loss=0.07193971 grad_norm=0.260246 alpha=1.000000 seconds=33.770
+step=14 loss=0.11812563 grad_norm=0.276972 alpha=1.000000 seconds=33.657
+step=15 loss=0.02560218 grad_norm=0.284253 alpha=1.000000 seconds=33.792
+step=16 loss=0.07697262 grad_norm=0.262389 alpha=1.000000 seconds=33.653
+step=17 loss=0.17103545 grad_norm=0.310579 alpha=1.000000 seconds=33.813
+step=18 loss=0.06112613 grad_norm=0.259268 alpha=1.000000 seconds=33.743
+step=19 loss=0.05431403 grad_norm=0.323605 alpha=1.000000 seconds=33.711
+step=20 loss=0.07360448 grad_norm=0.220407 alpha=1.000000 seconds=33.787
+step=21 loss=0.04034115 grad_norm=0.286446 alpha=1.000000 seconds=33.678
+step=22 loss=0.11012536 grad_norm=0.293315 alpha=1.000000 seconds=33.781
+step=23 loss=0.05604060 grad_norm=0.232609 alpha=1.000000 seconds=33.755
+step=24 loss=0.03554075 grad_norm=0.258779 alpha=1.000000 seconds=33.640
+step=25 loss=0.03441228 grad_norm=0.267229 alpha=1.000000 seconds=33.781
+step=26 loss=0.05237482 grad_norm=0.224972 alpha=1.000000 seconds=33.645
+step=27 loss=0.10940911 grad_norm=0.227375 alpha=1.000000 seconds=33.763
+step=28 loss=0.04694235 grad_norm=0.197525 alpha=1.000000 seconds=33.656
+step=29 loss=0.10112184 grad_norm=0.209473 alpha=1.000000 seconds=33.802
+step=30 loss=0.04324680 grad_norm=0.279391 alpha=1.000000 seconds=33.761
+step=31 loss=0.12733082 grad_norm=0.300622 alpha=1.000000 seconds=33.647
+step=32 loss=0.02403416 grad_norm=0.227129 alpha=1.000000 seconds=33.800
+step=33 loss=0.03110117 grad_norm=0.286761 alpha=1.000000 seconds=33.668
+step=34 loss=0.08550019 grad_norm=0.220576 alpha=1.000000 seconds=33.794
+step=35 loss=0.04438034 grad_norm=0.189675 alpha=1.000000 seconds=33.674
+step=36 loss=0.09064243 grad_norm=0.269070 alpha=1.000000 seconds=33.774
+step=37 loss=0.07698438 grad_norm=0.242023 alpha=1.000000 seconds=33.803
+step=38 loss=0.06099834 grad_norm=0.236653 alpha=1.000000 seconds=33.658
+step=39 loss=0.03180654 grad_norm=0.204874 alpha=1.000000 seconds=33.760
+step=40 loss=0.03417248 grad_norm=0.204233 alpha=1.000000 seconds=33.657
+step=41 loss=0.06671734 grad_norm=0.289587 alpha=1.000000 seconds=33.784
+step=42 loss=0.04921148 grad_norm=0.182852 alpha=1.000000 seconds=33.806
+step=43 loss=0.02129121 grad_norm=0.342663 alpha=1.000000 seconds=33.708
+step=44 loss=0.05286342 grad_norm=0.195751 alpha=1.000000 seconds=34.033
+step=45 loss=0.04283634 grad_norm=0.201901 alpha=1.000000 seconds=33.697
+step=46 loss=0.04427662 grad_norm=0.242933 alpha=1.000000 seconds=33.824
+step=47 loss=0.04108242 grad_norm=0.263488 alpha=1.000000 seconds=33.884
+step=48 loss=0.07543614 grad_norm=0.206035 alpha=1.000000 seconds=33.759
+step=49 loss=0.03876279 grad_norm=0.189900 alpha=1.000000 seconds=33.909
+step=50 loss=0.07528506 grad_norm=0.233832 alpha=1.000000 seconds=33.757
+step=51 loss=0.03334013 grad_norm=0.224337 alpha=1.000000 seconds=33.865
+step=52 loss=0.03060885 grad_norm=0.272599 alpha=1.000000 seconds=33.896
+step=53 loss=0.04899699 grad_norm=0.215172 alpha=1.000000 seconds=33.779
+step=54 loss=0.03130477 grad_norm=0.173256 alpha=1.000000 seconds=33.896
+step=55 loss=0.07736199 grad_norm=0.187667 alpha=1.000000 seconds=33.779
+step=56 loss=0.07770766 grad_norm=0.184238 alpha=1.000000 seconds=33.926
+step=57 loss=0.03969619 grad_norm=0.187981 alpha=1.000000 seconds=33.960
+step=58 loss=0.03687791 grad_norm=0.155044 alpha=1.000000 seconds=33.801
+step=59 loss=0.12523617 grad_norm=0.221522 alpha=1.000000 seconds=33.929
+step=60 loss=0.08622658 grad_norm=0.177215 alpha=1.000000 seconds=33.813
+step=61 loss=0.07227273 grad_norm=0.205775 alpha=1.000000 seconds=33.921
+step=62 loss=0.10847130 grad_norm=0.198043 alpha=1.000000 seconds=33.925
+step=63 loss=0.07740872 grad_norm=0.165815 alpha=1.000000 seconds=33.827
+step=64 loss=0.06024752 grad_norm=0.165368 alpha=1.000000 seconds=33.985
+step=65 loss=0.02398201 grad_norm=0.183096 alpha=1.000000 seconds=33.829
+step=66 loss=0.02832737 grad_norm=0.180772 alpha=1.000000 seconds=33.933
+step=67 loss=0.01544908 grad_norm=0.136101 alpha=1.000000 seconds=33.911
+step=68 loss=0.11497808 grad_norm=0.201052 alpha=1.000000 seconds=33.798
+step=69 loss=0.07206361 grad_norm=0.251406 alpha=1.000000 seconds=33.918
+step=70 loss=0.04051882 grad_norm=0.183865 alpha=1.000000 seconds=33.730
+step=71 loss=0.03630771 grad_norm=0.180706 alpha=1.000000 seconds=33.863
+step=72 loss=0.04538488 grad_norm=0.130744 alpha=1.000000 seconds=33.875
+step=73 loss=0.18013623 grad_norm=0.286744 alpha=1.000000 seconds=33.754
+step=74 loss=0.07830165 grad_norm=0.199704 alpha=1.000000 seconds=33.844
+step=75 loss=0.09745190 grad_norm=0.158924 alpha=1.000000 seconds=33.715
+step=76 loss=0.02690083 grad_norm=0.177477 alpha=1.000000 seconds=33.824
+step=77 loss=0.04714613 grad_norm=0.160328 alpha=1.000000 seconds=33.884
+step=78 loss=0.07830883 grad_norm=0.143745 alpha=1.000000 seconds=33.765
+step=79 loss=0.05118145 grad_norm=0.173350 alpha=1.000000 seconds=33.906
+step=80 loss=0.05994939 grad_norm=0.212281 alpha=1.000000 seconds=33.756
+step=81 loss=0.05339732 grad_norm=0.112174 alpha=1.000000 seconds=33.827
+step=82 loss=0.03782009 grad_norm=0.152065 alpha=1.000000 seconds=33.874
+step=83 loss=0.04048533 grad_norm=0.141790 alpha=1.000000 seconds=33.734
+step=84 loss=0.12918428 grad_norm=0.159513 alpha=1.000000 seconds=33.853
+step=85 loss=0.03003350 grad_norm=0.178977 alpha=1.000000 seconds=33.741
+step=86 loss=0.02502121 grad_norm=0.142148 alpha=1.000000 seconds=33.786
+step=87 loss=0.03851255 grad_norm=0.180289 alpha=1.000000 seconds=33.875
+step=88 loss=0.06391578 grad_norm=0.177133 alpha=1.000000 seconds=33.727
+step=89 loss=0.14157489 grad_norm=0.157599 alpha=1.000000 seconds=33.846
+step=90 loss=0.09798670 grad_norm=0.172020 alpha=1.000000 seconds=33.725
+step=91 loss=0.04864128 grad_norm=0.156068 alpha=1.000000 seconds=33.812
+step=92 loss=0.02191806 grad_norm=0.152512 alpha=1.000000 seconds=33.878
+step=93 loss=0.02280365 grad_norm=0.150145 alpha=1.000000 seconds=33.712
+step=94 loss=0.13381891 grad_norm=0.156895 alpha=1.000000 seconds=33.877
+step=95 loss=0.02748231 grad_norm=0.138590 alpha=1.000000 seconds=33.720
+step=96 loss=0.18177110 grad_norm=0.160178 alpha=1.000000 seconds=33.818
+step=97 loss=0.12929425 grad_norm=0.149322 alpha=1.000000 seconds=33.837
+step=98 loss=0.08150952 grad_norm=0.234453 alpha=1.000000 seconds=33.701
+step=99 loss=0.04211010 grad_norm=0.159517 alpha=1.000000 seconds=33.885
+step=100 loss=0.07087896 grad_norm=0.133036 alpha=1.000000 seconds=33.708
+validation step=100 loss=0.05426692 alpha=1.000000 samples=32 seconds=23.308
+Start gathering distributed model states...
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states.../home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/checkpoint_model_000100/model.pt sha256=1a2cdd3720e9823e69bc3542e83e0758e3e5ce235fd1d02fd12612259f04bb5f
+step=101 loss=0.06598934 grad_norm=0.151736 alpha=1.000000 seconds=33.348
+step=102 loss=0.03636648 grad_norm=0.098251 alpha=1.000000 seconds=33.861
+step=103 loss=0.07355691 grad_norm=0.117551 alpha=1.000000 seconds=33.708
+step=104 loss=0.17016563 grad_norm=0.194230 alpha=1.000000 seconds=33.812
+step=105 loss=0.02506783 grad_norm=0.111818 alpha=1.000000 seconds=33.811
+step=106 loss=0.02399110 grad_norm=0.192863 alpha=1.000000 seconds=33.683
+step=107 loss=0.03660506 grad_norm=0.181416 alpha=1.000000 seconds=33.818
+step=108 loss=0.06496543 grad_norm=0.152216 alpha=1.000000 seconds=33.683
+step=109 loss=0.21901207 grad_norm=0.152342 alpha=1.000000 seconds=33.818
+step=110 loss=0.03370446 grad_norm=0.132154 alpha=1.000000 seconds=33.811
+step=111 loss=0.07334892 grad_norm=0.127081 alpha=1.000000 seconds=33.658
+step=112 loss=0.14056440 grad_norm=0.168610 alpha=1.000000 seconds=33.820
+step=113 loss=0.12715648 grad_norm=0.166384 alpha=1.000000 seconds=33.638
+step=114 loss=0.03582515 grad_norm=0.125232 alpha=1.000000 seconds=33.747
+step=115 loss=0.06556825 grad_norm=0.177971 alpha=1.000000 seconds=33.794
+step=116 loss=0.03226463 grad_norm=0.120807 alpha=1.000000 seconds=33.710
+step=117 loss=0.10382182 grad_norm=0.156549 alpha=1.000000 seconds=33.818
+step=118 loss=0.02656615 grad_norm=0.155612 alpha=1.000000 seconds=33.693
+step=119 loss=0.06108290 grad_norm=0.117536 alpha=1.000000 seconds=33.788
+step=120 loss=0.05684963 grad_norm=0.121028 alpha=1.000000 seconds=33.781
+step=121 loss=0.04113654 grad_norm=0.134589 alpha=1.000000 seconds=33.644
+step=122 loss=0.02564879 grad_norm=0.142542 alpha=1.000000 seconds=33.826
+step=123 loss=0.06946641 grad_norm=0.126264 alpha=1.000000 seconds=33.687
+step=124 loss=0.02737449 grad_norm=0.131791 alpha=1.000000 seconds=33.758
+step=125 loss=0.10430790 grad_norm=0.164164 alpha=1.000000 seconds=33.775
+step=126 loss=0.17030542 grad_norm=0.135620 alpha=1.000000 seconds=33.654
+step=127 loss=0.06782351 grad_norm=0.111048 alpha=1.000000 seconds=33.804
+step=128 loss=0.03878323 grad_norm=0.120311 alpha=1.000000 seconds=33.652
+step=129 loss=0.06950964 grad_norm=0.142369 alpha=1.000000 seconds=33.800
+step=130 loss=0.05364748 grad_norm=0.124189 alpha=1.000000 seconds=33.809
+step=131 loss=0.06682634 grad_norm=0.122034 alpha=1.000000 seconds=33.643
+step=132 loss=0.03753212 grad_norm=0.133478 alpha=1.000000 seconds=33.805
+step=133 loss=0.10173719 grad_norm=0.127620 alpha=1.000000 seconds=33.636
+step=134 loss=0.09489484 grad_norm=0.156448 alpha=1.000000 seconds=33.740
+step=135 loss=0.03363843 grad_norm=0.107058 alpha=1.000000 seconds=33.752
+step=136 loss=0.05204394 grad_norm=0.114580 alpha=1.000000 seconds=33.648
+step=137 loss=0.09809074 grad_norm=0.160193 alpha=1.000000 seconds=33.789
+step=138 loss=0.12792566 grad_norm=0.116424 alpha=1.000000 seconds=33.658
+step=139 loss=0.04145581 grad_norm=0.121304 alpha=1.000000 seconds=33.758
+step=140 loss=0.01652843 grad_norm=0.150968 alpha=1.000000 seconds=33.769
+step=141 loss=0.05025722 grad_norm=0.121985 alpha=1.000000 seconds=33.635
+step=142 loss=0.10410172 grad_norm=0.135448 alpha=1.000000 seconds=33.833
+step=143 loss=0.03250777 grad_norm=0.160854 alpha=1.000000 seconds=33.664
+step=144 loss=0.02670942 grad_norm=0.182537 alpha=1.000000 seconds=33.776
+step=145 loss=0.07573386 grad_norm=0.108486 alpha=1.000000 seconds=33.764
+step=146 loss=0.02065845 grad_norm=0.198334 alpha=1.000000 seconds=33.684
+step=147 loss=0.05339679 grad_norm=0.108339 alpha=1.000000 seconds=33.800
+step=148 loss=0.07563239 grad_norm=0.100688 alpha=1.000000 seconds=33.676
+step=149 loss=0.03634264 grad_norm=0.114731 alpha=1.000000 seconds=33.746
+step=150 loss=0.07164769 grad_norm=0.120344 alpha=1.000000 seconds=33.774
+step=151 loss=0.03615677 grad_norm=0.117459 alpha=1.000000 seconds=33.665
+step=152 loss=0.15219596 grad_norm=0.126773 alpha=1.000000 seconds=33.847
+step=153 loss=0.03379047 grad_norm=0.079960 alpha=1.000000 seconds=33.675
+step=154 loss=0.01670950 grad_norm=0.095955 alpha=1.000000 seconds=33.771
+step=155 loss=0.03855304 grad_norm=0.094937 alpha=1.000000 seconds=33.766
+step=156 loss=0.03573131 grad_norm=0.120581 alpha=1.000000 seconds=33.650
+step=157 loss=0.03583978 grad_norm=0.096365 alpha=1.000000 seconds=33.836
+step=158 loss=0.05654490 grad_norm=0.121113 alpha=1.000000 seconds=33.644
+step=159 loss=0.02127133 grad_norm=0.082581 alpha=1.000000 seconds=33.755
+step=160 loss=0.02565955 grad_norm=0.091655 alpha=1.000000 seconds=33.761
+step=161 loss=0.05065660 grad_norm=0.107197 alpha=1.000000 seconds=33.664
+step=162 loss=0.06935982 grad_norm=0.093718 alpha=1.000000 seconds=33.826
+step=163 loss=0.03864646 grad_norm=0.089162 alpha=1.000000 seconds=33.680
+step=164 loss=0.02418797 grad_norm=0.098792 alpha=1.000000 seconds=33.761
+step=165 loss=0.06005814 grad_norm=0.117282 alpha=1.000000 seconds=33.694
+step=166 loss=0.02984983 grad_norm=0.129117 alpha=1.000000 seconds=33.641
+step=167 loss=0.03967089 grad_norm=0.132444 alpha=1.000000 seconds=33.812
+step=168 loss=0.01714222 grad_norm=0.120284 alpha=1.000000 seconds=33.649
+step=169 loss=0.09519900 grad_norm=0.101277 alpha=1.000000 seconds=33.755
+step=170 loss=0.07254214 grad_norm=0.099185 alpha=1.000000 seconds=33.772
+step=171 loss=0.04403805 grad_norm=0.083316 alpha=1.000000 seconds=33.656
+step=172 loss=0.02450114 grad_norm=0.103612 alpha=1.000000 seconds=33.842
+step=173 loss=0.12422589 grad_norm=0.117701 alpha=1.000000 seconds=33.685
+step=174 loss=0.03805575 grad_norm=0.091695 alpha=1.000000 seconds=33.812
+step=175 loss=0.09506878 grad_norm=0.108078 alpha=1.000000 seconds=33.835
+step=176 loss=0.04276789 grad_norm=0.104158 alpha=1.000000 seconds=33.712
+step=177 loss=0.08150393 grad_norm=0.112759 alpha=1.000000 seconds=33.874
+step=178 loss=0.14357159 grad_norm=0.127649 alpha=1.000000 seconds=33.744
+step=179 loss=0.04571144 grad_norm=0.082729 alpha=1.000000 seconds=33.877
+step=180 loss=0.04081282 grad_norm=0.109078 alpha=1.000000 seconds=33.875
+step=181 loss=0.03135540 grad_norm=0.093390 alpha=1.000000 seconds=33.753
+step=182 loss=0.10407751 grad_norm=0.121781 alpha=1.000000 seconds=33.911
+step=183 loss=0.03013783 grad_norm=0.082267 alpha=1.000000 seconds=33.786
+step=184 loss=0.03965293 grad_norm=0.094319 alpha=1.000000 seconds=33.829
+step=185 loss=0.05627605 grad_norm=0.122151 alpha=1.000000 seconds=33.889
+step=186 loss=0.04662800 grad_norm=0.099338 alpha=1.000000 seconds=33.772
+step=187 loss=0.04081826 grad_norm=0.084307 alpha=1.000000 seconds=33.933
+step=188 loss=0.06218822 grad_norm=0.103257 alpha=1.000000 seconds=33.814
+step=189 loss=0.03355948 grad_norm=0.132526 alpha=1.000000 seconds=33.914
+step=190 loss=0.07602763 grad_norm=0.137893 alpha=1.000000 seconds=33.926
+step=191 loss=0.03005564 grad_norm=0.078515 alpha=1.000000 seconds=33.825
+step=192 loss=0.02428656 grad_norm=0.083356 alpha=1.000000 seconds=33.985
+step=193 loss=0.05848589 grad_norm=0.133774 alpha=1.000000 seconds=33.797
+step=194 loss=0.08737557 grad_norm=0.112545 alpha=1.000000 seconds=33.948
+step=195 loss=0.06110434 grad_norm=0.115061 alpha=1.000000 seconds=33.936
+step=196 loss=0.04816964 grad_norm=0.103439 alpha=1.000000 seconds=33.830
+step=197 loss=0.08123685 grad_norm=0.110852 alpha=1.000000 seconds=33.961
+step=198 loss=0.12577854 grad_norm=0.144993 alpha=1.000000 seconds=33.776
+step=199 loss=0.02403097 grad_norm=0.124394 alpha=1.000000 seconds=33.904
+step=200 loss=0.03727136 grad_norm=0.104435 alpha=1.000000 seconds=33.884
+validation step=200 loss=0.05151268 alpha=1.000000 samples=32 seconds=23.285
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/checkpoint_model_000200/model.pt sha256=026104aeeaa41d4df33f2d6289a002e734f286ac4ed160ccd7bcb33dbb906b57

outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/training_metrics.jsonl

@@ -0,0 +1,200 @@
+{"step": 1, "generator_loss": 0.16898617148399353, "generator_grad_norm": 0.28445854783058167, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80415177345276, "peak_memory_bytes": 20785906176}
+{"step": 2, "generator_loss": 0.08548039942979813, "generator_grad_norm": 0.2637227773666382, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.384560108184814, "peak_memory_bytes": 22206258176}
+{"step": 3, "generator_loss": 0.07998532801866531, "generator_grad_norm": 0.32123035192489624, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.639726638793945, "peak_memory_bytes": 22206258176}
+{"step": 4, "generator_loss": 0.11531127244234085, "generator_grad_norm": 0.292703777551651, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.58222246170044, "peak_memory_bytes": 22206258176}
+{"step": 5, "generator_loss": 0.04095565900206566, "generator_grad_norm": 0.4411107003688812, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.763837814331055, "peak_memory_bytes": 22206694400}
+{"step": 6, "generator_loss": 0.05031534284353256, "generator_grad_norm": 0.24605269730091095, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77384567260742, "peak_memory_bytes": 22206694400}
+{"step": 7, "generator_loss": 0.12238266319036484, "generator_grad_norm": 0.30054375529289246, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.60242176055908, "peak_memory_bytes": 22206694400}
+{"step": 8, "generator_loss": 0.04164436459541321, "generator_grad_norm": 0.28025656938552856, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.74710822105408, "peak_memory_bytes": 22206694400}
+{"step": 9, "generator_loss": 0.04078318551182747, "generator_grad_norm": 0.29020851850509644, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.663604497909546, "peak_memory_bytes": 22206694400}
+{"step": 10, "generator_loss": 0.11827541887760162, "generator_grad_norm": 0.361130952835083, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.76335144042969, "peak_memory_bytes": 22206694400}
+{"step": 11, "generator_loss": 0.05484040826559067, "generator_grad_norm": 0.29593223333358765, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.6941192150116, "peak_memory_bytes": 22206694400}
+{"step": 12, "generator_loss": 0.04062698781490326, "generator_grad_norm": 0.22531741857528687, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.643850803375244, "peak_memory_bytes": 22206694400}
+{"step": 13, "generator_loss": 0.07193970680236816, "generator_grad_norm": 0.2602464258670807, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77005624771118, "peak_memory_bytes": 22206694400}
+{"step": 14, "generator_loss": 0.11812563240528107, "generator_grad_norm": 0.27697211503982544, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.65667176246643, "peak_memory_bytes": 22206694400}
+{"step": 15, "generator_loss": 0.025602182373404503, "generator_grad_norm": 0.28425332903862, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79198741912842, "peak_memory_bytes": 22206694400}
+{"step": 16, "generator_loss": 0.07697261869907379, "generator_grad_norm": 0.26238879561424255, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.6530499458313, "peak_memory_bytes": 22206694400}
+{"step": 17, "generator_loss": 0.17103545367717743, "generator_grad_norm": 0.3105790317058563, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8133327960968, "peak_memory_bytes": 22206694400}
+{"step": 18, "generator_loss": 0.06112613156437874, "generator_grad_norm": 0.2592679560184479, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.742992639541626, "peak_memory_bytes": 22206694400}
+{"step": 19, "generator_loss": 0.0543140284717083, "generator_grad_norm": 0.3236050307750702, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.710947036743164, "peak_memory_bytes": 22206694400}
+{"step": 20, "generator_loss": 0.07360447943210602, "generator_grad_norm": 0.22040706872940063, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.786736726760864, "peak_memory_bytes": 22206831616}
+{"step": 21, "generator_loss": 0.04034114629030228, "generator_grad_norm": 0.2864460051059723, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.67788052558899, "peak_memory_bytes": 22206831616}
+{"step": 22, "generator_loss": 0.11012536287307739, "generator_grad_norm": 0.2933145761489868, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.780598878860474, "peak_memory_bytes": 22206831616}
+{"step": 23, "generator_loss": 0.05604059621691704, "generator_grad_norm": 0.2326088696718216, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75452756881714, "peak_memory_bytes": 22206981120}
+{"step": 24, "generator_loss": 0.03554074838757515, "generator_grad_norm": 0.2587786614894867, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.639894008636475, "peak_memory_bytes": 22206981120}
+{"step": 25, "generator_loss": 0.034412283450365067, "generator_grad_norm": 0.2672293782234192, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.781068086624146, "peak_memory_bytes": 22206981120}
+{"step": 26, "generator_loss": 0.05237481743097305, "generator_grad_norm": 0.22497165203094482, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.64522886276245, "peak_memory_bytes": 22206981120}
+{"step": 27, "generator_loss": 0.10940910875797272, "generator_grad_norm": 0.22737464308738708, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7633843421936, "peak_memory_bytes": 22206981120}
+{"step": 28, "generator_loss": 0.04694234952330589, "generator_grad_norm": 0.19752490520477295, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.656442165374756, "peak_memory_bytes": 22206981120}
+{"step": 29, "generator_loss": 0.10112183541059494, "generator_grad_norm": 0.20947276055812836, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80181956291199, "peak_memory_bytes": 22206981120}
+{"step": 30, "generator_loss": 0.0432467982172966, "generator_grad_norm": 0.2793906033039093, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7607421875, "peak_memory_bytes": 22206981120}
+{"step": 31, "generator_loss": 0.12733082473278046, "generator_grad_norm": 0.3006223142147064, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.6474506855011, "peak_memory_bytes": 22206981120}
+{"step": 32, "generator_loss": 0.0240341629832983, "generator_grad_norm": 0.22712911665439606, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79999303817749, "peak_memory_bytes": 22206981120}
+{"step": 33, "generator_loss": 0.0311011653393507, "generator_grad_norm": 0.28676101565361023, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.668256759643555, "peak_memory_bytes": 22206981120}
+{"step": 34, "generator_loss": 0.08550018817186356, "generator_grad_norm": 0.22057592868804932, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79374670982361, "peak_memory_bytes": 22206981120}
+{"step": 35, "generator_loss": 0.04438033699989319, "generator_grad_norm": 0.18967507779598236, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.674049615859985, "peak_memory_bytes": 22206981120}
+{"step": 36, "generator_loss": 0.09064242988824844, "generator_grad_norm": 0.2690700590610504, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77378702163696, "peak_memory_bytes": 22206981120}
+{"step": 37, "generator_loss": 0.07698438316583633, "generator_grad_norm": 0.24202275276184082, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80255126953125, "peak_memory_bytes": 22206981120}
+{"step": 38, "generator_loss": 0.0609983429312706, "generator_grad_norm": 0.2366534322500229, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.657522678375244, "peak_memory_bytes": 22206981120}
+{"step": 39, "generator_loss": 0.03180653601884842, "generator_grad_norm": 0.20487384498119354, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.759639501571655, "peak_memory_bytes": 22206981120}
+{"step": 40, "generator_loss": 0.034172482788562775, "generator_grad_norm": 0.20423267781734467, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.65740966796875, "peak_memory_bytes": 22206981120}
+{"step": 41, "generator_loss": 0.06671734154224396, "generator_grad_norm": 0.28958660364151, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.78388524055481, "peak_memory_bytes": 22206981120}
+{"step": 42, "generator_loss": 0.04921147599816322, "generator_grad_norm": 0.18285170197486877, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80577850341797, "peak_memory_bytes": 22206981120}
+{"step": 43, "generator_loss": 0.021291211247444153, "generator_grad_norm": 0.3426632285118103, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.70793128013611, "peak_memory_bytes": 22206981120}
+{"step": 44, "generator_loss": 0.05286341905593872, "generator_grad_norm": 0.19575060904026031, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.032790422439575, "peak_memory_bytes": 22206981120}
+{"step": 45, "generator_loss": 0.04283634200692177, "generator_grad_norm": 0.20190118253231049, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.69670915603638, "peak_memory_bytes": 22206981120}
+{"step": 46, "generator_loss": 0.04427661746740341, "generator_grad_norm": 0.2429332137107849, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82408881187439, "peak_memory_bytes": 22206981120}
+{"step": 47, "generator_loss": 0.041082415729761124, "generator_grad_norm": 0.2634880244731903, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88358783721924, "peak_memory_bytes": 22207118336}
+{"step": 48, "generator_loss": 0.07543613761663437, "generator_grad_norm": 0.20603522658348083, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.759230613708496, "peak_memory_bytes": 22207118336}
+{"step": 49, "generator_loss": 0.03876279294490814, "generator_grad_norm": 0.1898995041847229, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.90921115875244, "peak_memory_bytes": 22207118336}
+{"step": 50, "generator_loss": 0.07528506219387054, "generator_grad_norm": 0.23383165895938873, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75728225708008, "peak_memory_bytes": 22207118336}
+{"step": 51, "generator_loss": 0.03334013372659683, "generator_grad_norm": 0.22433707118034363, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.86546754837036, "peak_memory_bytes": 22207118336}
+{"step": 52, "generator_loss": 0.030608853325247765, "generator_grad_norm": 0.2725989818572998, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.89612531661987, "peak_memory_bytes": 22207118336}
+{"step": 53, "generator_loss": 0.04899698868393898, "generator_grad_norm": 0.21517153084278107, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.778764724731445, "peak_memory_bytes": 22207118336}
+{"step": 54, "generator_loss": 0.031304772943258286, "generator_grad_norm": 0.17325586080551147, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.89599418640137, "peak_memory_bytes": 22207118336}
+{"step": 55, "generator_loss": 0.07736199349164963, "generator_grad_norm": 0.18766739964485168, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77913784980774, "peak_memory_bytes": 22207118336}
+{"step": 56, "generator_loss": 0.07770766317844391, "generator_grad_norm": 0.18423812091350555, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.926034927368164, "peak_memory_bytes": 22207118336}
+{"step": 57, "generator_loss": 0.039696190506219864, "generator_grad_norm": 0.18798083066940308, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.960367918014526, "peak_memory_bytes": 22207118336}
+{"step": 58, "generator_loss": 0.036877911537885666, "generator_grad_norm": 0.15504425764083862, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80120873451233, "peak_memory_bytes": 22207118336}
+{"step": 59, "generator_loss": 0.1252361685037613, "generator_grad_norm": 0.2215222716331482, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.929280281066895, "peak_memory_bytes": 22207118336}
+{"step": 60, "generator_loss": 0.08622657507658005, "generator_grad_norm": 0.17721502482891083, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8127875328064, "peak_memory_bytes": 22207118336}
+{"step": 61, "generator_loss": 0.07227273285388947, "generator_grad_norm": 0.20577527582645416, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.92121648788452, "peak_memory_bytes": 22207118336}
+{"step": 62, "generator_loss": 0.10847130417823792, "generator_grad_norm": 0.19804342091083527, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.925373554229736, "peak_memory_bytes": 22207118336}
+{"step": 63, "generator_loss": 0.07740871608257294, "generator_grad_norm": 0.16581466794013977, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82723045349121, "peak_memory_bytes": 22207118336}
+{"step": 64, "generator_loss": 0.0602475181221962, "generator_grad_norm": 0.1653677374124527, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.98465585708618, "peak_memory_bytes": 22207118336}
+{"step": 65, "generator_loss": 0.023982005193829536, "generator_grad_norm": 0.18309596180915833, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.829299211502075, "peak_memory_bytes": 22207118336}
+{"step": 66, "generator_loss": 0.028327371925115585, "generator_grad_norm": 0.1807715892791748, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.933135747909546, "peak_memory_bytes": 22207118336}
+{"step": 67, "generator_loss": 0.015449078753590584, "generator_grad_norm": 0.13610120117664337, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9105544090271, "peak_memory_bytes": 22207118336}
+{"step": 68, "generator_loss": 0.11497807502746582, "generator_grad_norm": 0.20105238258838654, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79821038246155, "peak_memory_bytes": 22207118336}
+{"step": 69, "generator_loss": 0.07206360995769501, "generator_grad_norm": 0.2514064311981201, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.918235301971436, "peak_memory_bytes": 22207118336}
+{"step": 70, "generator_loss": 0.04051882401108742, "generator_grad_norm": 0.18386481702327728, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.73016977310181, "peak_memory_bytes": 22207118336}
+{"step": 71, "generator_loss": 0.03630770742893219, "generator_grad_norm": 0.18070591986179352, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.862672567367554, "peak_memory_bytes": 22207118336}
+{"step": 72, "generator_loss": 0.04538487643003464, "generator_grad_norm": 0.13074427843093872, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.87534189224243, "peak_memory_bytes": 22207118336}
+{"step": 73, "generator_loss": 0.18013623356819153, "generator_grad_norm": 0.28674405813217163, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75443077087402, "peak_memory_bytes": 22207118336}
+{"step": 74, "generator_loss": 0.07830164581537247, "generator_grad_norm": 0.19970418512821198, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.843674659729004, "peak_memory_bytes": 22207118336}
+{"step": 75, "generator_loss": 0.09745189547538757, "generator_grad_norm": 0.15892377495765686, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7153525352478, "peak_memory_bytes": 22207118336}
+{"step": 76, "generator_loss": 0.026900826022028923, "generator_grad_norm": 0.17747719585895538, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82384991645813, "peak_memory_bytes": 22207118336}
+{"step": 77, "generator_loss": 0.04714612662792206, "generator_grad_norm": 0.16032817959785461, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88415884971619, "peak_memory_bytes": 22207118336}
+{"step": 78, "generator_loss": 0.0783088281750679, "generator_grad_norm": 0.14374525845050812, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.764766216278076, "peak_memory_bytes": 22207118336}
+{"step": 79, "generator_loss": 0.051181454211473465, "generator_grad_norm": 0.1733495444059372, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.90575361251831, "peak_memory_bytes": 22207118336}
+{"step": 80, "generator_loss": 0.05994938686490059, "generator_grad_norm": 0.21228130161762238, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.756221294403076, "peak_memory_bytes": 22207118336}
+{"step": 81, "generator_loss": 0.05339731648564339, "generator_grad_norm": 0.11217417567968369, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82657241821289, "peak_memory_bytes": 22207118336}
+{"step": 82, "generator_loss": 0.03782009333372116, "generator_grad_norm": 0.15206491947174072, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8739652633667, "peak_memory_bytes": 22207118336}
+{"step": 83, "generator_loss": 0.04048532992601395, "generator_grad_norm": 0.14178955554962158, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.73436236381531, "peak_memory_bytes": 22207118336}
+{"step": 84, "generator_loss": 0.1291842758655548, "generator_grad_norm": 0.1595132201910019, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.853405237197876, "peak_memory_bytes": 22207118336}
+{"step": 85, "generator_loss": 0.030033500865101814, "generator_grad_norm": 0.17897656559944153, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.741148233413696, "peak_memory_bytes": 22207118336}
+{"step": 86, "generator_loss": 0.025021206587553024, "generator_grad_norm": 0.1421481817960739, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.785653591156006, "peak_memory_bytes": 22207118336}
+{"step": 87, "generator_loss": 0.03851255029439926, "generator_grad_norm": 0.18028868734836578, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.874906063079834, "peak_memory_bytes": 22207118336}
+{"step": 88, "generator_loss": 0.06391578167676926, "generator_grad_norm": 0.17713303864002228, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.727179765701294, "peak_memory_bytes": 22207118336}
+{"step": 89, "generator_loss": 0.141574889421463, "generator_grad_norm": 0.15759903192520142, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.845869302749634, "peak_memory_bytes": 22207118336}
+{"step": 90, "generator_loss": 0.09798669815063477, "generator_grad_norm": 0.17201995849609375, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.72495627403259, "peak_memory_bytes": 22207118336}
+{"step": 91, "generator_loss": 0.048641275614500046, "generator_grad_norm": 0.1560676544904709, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81234955787659, "peak_memory_bytes": 22207118336}
+{"step": 92, "generator_loss": 0.021918056532740593, "generator_grad_norm": 0.15251237154006958, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.877782106399536, "peak_memory_bytes": 22207118336}
+{"step": 93, "generator_loss": 0.0228036530315876, "generator_grad_norm": 0.15014459192752838, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.71194863319397, "peak_memory_bytes": 22207118336}
+{"step": 94, "generator_loss": 0.13381890952587128, "generator_grad_norm": 0.15689517557621002, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.876649618148804, "peak_memory_bytes": 22207118336}
+{"step": 95, "generator_loss": 0.027482306584715843, "generator_grad_norm": 0.13859008252620697, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.72015738487244, "peak_memory_bytes": 22207118336}
+{"step": 96, "generator_loss": 0.18177109956741333, "generator_grad_norm": 0.1601780503988266, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81753993034363, "peak_memory_bytes": 22207118336}
+{"step": 97, "generator_loss": 0.1292942464351654, "generator_grad_norm": 0.149321511387825, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.83658313751221, "peak_memory_bytes": 22207118336}
+{"step": 98, "generator_loss": 0.08150952309370041, "generator_grad_norm": 0.23445278406143188, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.70086121559143, "peak_memory_bytes": 22207118336}
+{"step": 99, "generator_loss": 0.04211009666323662, "generator_grad_norm": 0.15951679646968842, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88490557670593, "peak_memory_bytes": 22207118336}
+{"step": 100, "generator_loss": 0.07087896019220352, "generator_grad_norm": 0.13303641974925995, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.70848989486694, "peak_memory_bytes": 22207118336}
+{"step": 101, "generator_loss": 0.06598933786153793, "generator_grad_norm": 0.15173625946044922, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.34785437583923, "peak_memory_bytes": 22207118336}
+{"step": 102, "generator_loss": 0.036366481333971024, "generator_grad_norm": 0.09825082868337631, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.86149001121521, "peak_memory_bytes": 22207118336}
+{"step": 103, "generator_loss": 0.07355690747499466, "generator_grad_norm": 0.11755065619945526, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.70844531059265, "peak_memory_bytes": 22207118336}
+{"step": 104, "generator_loss": 0.17016562819480896, "generator_grad_norm": 0.19423040747642517, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81212544441223, "peak_memory_bytes": 22207118336}
+{"step": 105, "generator_loss": 0.025067832320928574, "generator_grad_norm": 0.11181823164224625, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81146502494812, "peak_memory_bytes": 22207118336}
+{"step": 106, "generator_loss": 0.02399110235273838, "generator_grad_norm": 0.19286344945430756, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.682889223098755, "peak_memory_bytes": 22207118336}
+{"step": 107, "generator_loss": 0.03660506010055542, "generator_grad_norm": 0.18141593039035797, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.818328857421875, "peak_memory_bytes": 22207118336}
+{"step": 108, "generator_loss": 0.06496543437242508, "generator_grad_norm": 0.15221631526947021, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.68317794799805, "peak_memory_bytes": 22207118336}
+{"step": 109, "generator_loss": 0.2190120667219162, "generator_grad_norm": 0.15234224498271942, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81841969490051, "peak_memory_bytes": 22207118336}
+{"step": 110, "generator_loss": 0.03370445594191551, "generator_grad_norm": 0.13215424120426178, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81121563911438, "peak_memory_bytes": 22207118336}
+{"step": 111, "generator_loss": 0.07334892451763153, "generator_grad_norm": 0.1270812600851059, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.657732009887695, "peak_memory_bytes": 22207118336}
+{"step": 112, "generator_loss": 0.14056439697742462, "generator_grad_norm": 0.16861042380332947, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.820111989974976, "peak_memory_bytes": 22207118336}
+{"step": 113, "generator_loss": 0.12715648114681244, "generator_grad_norm": 0.16638368368148804, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.63759469985962, "peak_memory_bytes": 22207118336}
+{"step": 114, "generator_loss": 0.035825151950120926, "generator_grad_norm": 0.12523241341114044, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.74716353416443, "peak_memory_bytes": 22207118336}
+{"step": 115, "generator_loss": 0.06556824594736099, "generator_grad_norm": 0.17797143757343292, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79408669471741, "peak_memory_bytes": 22207118336}
+{"step": 116, "generator_loss": 0.03226463124155998, "generator_grad_norm": 0.12080670148134232, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.710153579711914, "peak_memory_bytes": 22207118336}
+{"step": 117, "generator_loss": 0.10382182151079178, "generator_grad_norm": 0.156549334526062, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.818222522735596, "peak_memory_bytes": 22207118336}
+{"step": 118, "generator_loss": 0.026566151529550552, "generator_grad_norm": 0.15561218559741974, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.69315242767334, "peak_memory_bytes": 22207118336}
+{"step": 119, "generator_loss": 0.06108289957046509, "generator_grad_norm": 0.11753565073013306, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.788097620010376, "peak_memory_bytes": 22207118336}
+{"step": 120, "generator_loss": 0.056849632412195206, "generator_grad_norm": 0.12102840840816498, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.78108859062195, "peak_memory_bytes": 22207118336}
+{"step": 121, "generator_loss": 0.04113653674721718, "generator_grad_norm": 0.1345887929201126, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.64355444908142, "peak_memory_bytes": 22207118336}
+{"step": 122, "generator_loss": 0.02564878948032856, "generator_grad_norm": 0.14254210889339447, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.825698137283325, "peak_memory_bytes": 22207118336}
+{"step": 123, "generator_loss": 0.06946641206741333, "generator_grad_norm": 0.12626376748085022, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.68745756149292, "peak_memory_bytes": 22207118336}
+{"step": 124, "generator_loss": 0.02737448923289776, "generator_grad_norm": 0.13179129362106323, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7580931186676, "peak_memory_bytes": 22207118336}
+{"step": 125, "generator_loss": 0.10430790483951569, "generator_grad_norm": 0.16416409611701965, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77526617050171, "peak_memory_bytes": 22207118336}
+{"step": 126, "generator_loss": 0.17030541598796844, "generator_grad_norm": 0.13561967015266418, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.6544394493103, "peak_memory_bytes": 22207118336}
+{"step": 127, "generator_loss": 0.06782351434230804, "generator_grad_norm": 0.11104818433523178, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80412983894348, "peak_memory_bytes": 22207118336}
+{"step": 128, "generator_loss": 0.0387832336127758, "generator_grad_norm": 0.12031079083681107, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.65195870399475, "peak_memory_bytes": 22207118336}
+{"step": 129, "generator_loss": 0.06950964033603668, "generator_grad_norm": 0.14236901700496674, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8004310131073, "peak_memory_bytes": 22207118336}
+{"step": 130, "generator_loss": 0.053647480905056, "generator_grad_norm": 0.12418949604034424, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.808706521987915, "peak_memory_bytes": 22207118336}
+{"step": 131, "generator_loss": 0.06682633608579636, "generator_grad_norm": 0.1220342218875885, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.64332699775696, "peak_memory_bytes": 22207118336}
+{"step": 132, "generator_loss": 0.037532124668359756, "generator_grad_norm": 0.1334778517484665, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.80451226234436, "peak_memory_bytes": 22207118336}
+{"step": 133, "generator_loss": 0.10173719376325607, "generator_grad_norm": 0.12761971354484558, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.63588309288025, "peak_memory_bytes": 22207118336}
+{"step": 134, "generator_loss": 0.09489484131336212, "generator_grad_norm": 0.15644849836826324, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.74011421203613, "peak_memory_bytes": 22207118336}
+{"step": 135, "generator_loss": 0.03363842889666557, "generator_grad_norm": 0.10705845803022385, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75227665901184, "peak_memory_bytes": 22207118336}
+{"step": 136, "generator_loss": 0.052043937146663666, "generator_grad_norm": 0.11457954347133636, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.648415088653564, "peak_memory_bytes": 22207118336}
+{"step": 137, "generator_loss": 0.09809073805809021, "generator_grad_norm": 0.16019266843795776, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.78930592536926, "peak_memory_bytes": 22207118336}
+{"step": 138, "generator_loss": 0.12792566418647766, "generator_grad_norm": 0.11642403900623322, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.658360719680786, "peak_memory_bytes": 22207118336}
+{"step": 139, "generator_loss": 0.04145580902695656, "generator_grad_norm": 0.12130353599786758, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.758161306381226, "peak_memory_bytes": 22207118336}
+{"step": 140, "generator_loss": 0.016528425738215446, "generator_grad_norm": 0.1509677916765213, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.769240617752075, "peak_memory_bytes": 22207118336}
+{"step": 141, "generator_loss": 0.05025721713900566, "generator_grad_norm": 0.12198548763990402, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.63549208641052, "peak_memory_bytes": 22207118336}
+{"step": 142, "generator_loss": 0.10410171747207642, "generator_grad_norm": 0.13544759154319763, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8332245349884, "peak_memory_bytes": 22207118336}
+{"step": 143, "generator_loss": 0.032507773488759995, "generator_grad_norm": 0.160853773355484, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.663989305496216, "peak_memory_bytes": 22207118336}
+{"step": 144, "generator_loss": 0.02670942060649395, "generator_grad_norm": 0.18253669142723083, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77618360519409, "peak_memory_bytes": 22207118336}
+{"step": 145, "generator_loss": 0.07573386281728745, "generator_grad_norm": 0.10848555713891983, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.76448464393616, "peak_memory_bytes": 22207118336}
+{"step": 146, "generator_loss": 0.020658450201153755, "generator_grad_norm": 0.19833357632160187, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.68424367904663, "peak_memory_bytes": 22207118336}
+{"step": 147, "generator_loss": 0.0533967949450016, "generator_grad_norm": 0.10833895951509476, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79981565475464, "peak_memory_bytes": 22207118336}
+{"step": 148, "generator_loss": 0.07563239336013794, "generator_grad_norm": 0.10068768262863159, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.67587614059448, "peak_memory_bytes": 22207118336}
+{"step": 149, "generator_loss": 0.036342643201351166, "generator_grad_norm": 0.11473137140274048, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.74603247642517, "peak_memory_bytes": 22207118336}
+{"step": 150, "generator_loss": 0.07164768874645233, "generator_grad_norm": 0.12034408003091812, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77404427528381, "peak_memory_bytes": 22207118336}
+{"step": 151, "generator_loss": 0.03615676984190941, "generator_grad_norm": 0.1174590140581131, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.66496753692627, "peak_memory_bytes": 22207118336}
+{"step": 152, "generator_loss": 0.15219596028327942, "generator_grad_norm": 0.12677323818206787, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.84685492515564, "peak_memory_bytes": 22207118336}
+{"step": 153, "generator_loss": 0.0337904691696167, "generator_grad_norm": 0.07995963096618652, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.67500948905945, "peak_memory_bytes": 22207118336}
+{"step": 154, "generator_loss": 0.016709502786397934, "generator_grad_norm": 0.09595528990030289, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.771448850631714, "peak_memory_bytes": 22207118336}
+{"step": 155, "generator_loss": 0.03855304419994354, "generator_grad_norm": 0.09493697434663773, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.76636219024658, "peak_memory_bytes": 22207118336}
+{"step": 156, "generator_loss": 0.03573131188750267, "generator_grad_norm": 0.12058111280202866, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.65012574195862, "peak_memory_bytes": 22207118336}
+{"step": 157, "generator_loss": 0.035839784890413284, "generator_grad_norm": 0.09636518359184265, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.83605909347534, "peak_memory_bytes": 22207118336}
+{"step": 158, "generator_loss": 0.05654489994049072, "generator_grad_norm": 0.12111302465200424, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.643885374069214, "peak_memory_bytes": 22207118336}
+{"step": 159, "generator_loss": 0.02127132937312126, "generator_grad_norm": 0.08258099853992462, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75547957420349, "peak_memory_bytes": 22207118336}
+{"step": 160, "generator_loss": 0.025659549981355667, "generator_grad_norm": 0.0916551947593689, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7605299949646, "peak_memory_bytes": 22207118336}
+{"step": 161, "generator_loss": 0.050656598061323166, "generator_grad_norm": 0.1071973517537117, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.66440939903259, "peak_memory_bytes": 22207118336}
+{"step": 162, "generator_loss": 0.06935981661081314, "generator_grad_norm": 0.09371782839298248, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82592487335205, "peak_memory_bytes": 22207118336}
+{"step": 163, "generator_loss": 0.038646455854177475, "generator_grad_norm": 0.08916184306144714, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.68029594421387, "peak_memory_bytes": 22207118336}
+{"step": 164, "generator_loss": 0.0241879653185606, "generator_grad_norm": 0.09879197925329208, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.76106929779053, "peak_memory_bytes": 22207118336}
+{"step": 165, "generator_loss": 0.06005813926458359, "generator_grad_norm": 0.11728168278932571, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.6944694519043, "peak_memory_bytes": 22207118336}
+{"step": 166, "generator_loss": 0.029849831014871597, "generator_grad_norm": 0.12911738455295563, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.64067840576172, "peak_memory_bytes": 22207118336}
+{"step": 167, "generator_loss": 0.03967088833451271, "generator_grad_norm": 0.13244397938251495, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.811973094940186, "peak_memory_bytes": 22207118336}
+{"step": 168, "generator_loss": 0.017142215743660927, "generator_grad_norm": 0.12028399854898453, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.649396657943726, "peak_memory_bytes": 22207118336}
+{"step": 169, "generator_loss": 0.09519899636507034, "generator_grad_norm": 0.10127732902765274, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.75466704368591, "peak_memory_bytes": 22207118336}
+{"step": 170, "generator_loss": 0.07254213839769363, "generator_grad_norm": 0.09918542206287384, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77196645736694, "peak_memory_bytes": 22207118336}
+{"step": 171, "generator_loss": 0.04403804615139961, "generator_grad_norm": 0.08331577479839325, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.655879497528076, "peak_memory_bytes": 22207118336}
+{"step": 172, "generator_loss": 0.024501144886016846, "generator_grad_norm": 0.10361247509717941, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.842488050460815, "peak_memory_bytes": 22207118336}
+{"step": 173, "generator_loss": 0.12422589212656021, "generator_grad_norm": 0.11770066618919373, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.68498110771179, "peak_memory_bytes": 22207118336}
+{"step": 174, "generator_loss": 0.038055747747421265, "generator_grad_norm": 0.0916949138045311, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.811790466308594, "peak_memory_bytes": 22207118336}
+{"step": 175, "generator_loss": 0.0950687825679779, "generator_grad_norm": 0.10807808488607407, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.83496832847595, "peak_memory_bytes": 22207118336}
+{"step": 176, "generator_loss": 0.04276788979768753, "generator_grad_norm": 0.10415764898061752, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.711506843566895, "peak_memory_bytes": 22207118336}
+{"step": 177, "generator_loss": 0.08150392770767212, "generator_grad_norm": 0.11275855451822281, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.87393140792847, "peak_memory_bytes": 22207118336}
+{"step": 178, "generator_loss": 0.14357158541679382, "generator_grad_norm": 0.12764893472194672, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.7441246509552, "peak_memory_bytes": 22207118336}
+{"step": 179, "generator_loss": 0.045711442828178406, "generator_grad_norm": 0.08272861689329147, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.877410650253296, "peak_memory_bytes": 22207118336}
+{"step": 180, "generator_loss": 0.04081282392144203, "generator_grad_norm": 0.10907842963933945, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.87476849555969, "peak_memory_bytes": 22207118336}
+{"step": 181, "generator_loss": 0.03135539963841438, "generator_grad_norm": 0.09338956326246262, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.753395557403564, "peak_memory_bytes": 22207118336}
+{"step": 182, "generator_loss": 0.10407751053571701, "generator_grad_norm": 0.12178121507167816, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9111692905426, "peak_memory_bytes": 22207118336}
+{"step": 183, "generator_loss": 0.03013783134520054, "generator_grad_norm": 0.08226655423641205, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.785948038101196, "peak_memory_bytes": 22207118336}
+{"step": 184, "generator_loss": 0.039652932435274124, "generator_grad_norm": 0.0943189263343811, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82891821861267, "peak_memory_bytes": 22207118336}
+{"step": 185, "generator_loss": 0.056276049464941025, "generator_grad_norm": 0.12215063720941544, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88899064064026, "peak_memory_bytes": 22207118336}
+{"step": 186, "generator_loss": 0.04662800207734108, "generator_grad_norm": 0.09933802485466003, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.77190279960632, "peak_memory_bytes": 22207118336}
+{"step": 187, "generator_loss": 0.040818262845277786, "generator_grad_norm": 0.08430738747119904, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93272256851196, "peak_memory_bytes": 22207118336}
+{"step": 188, "generator_loss": 0.06218821927905083, "generator_grad_norm": 0.10325734317302704, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.81364607810974, "peak_memory_bytes": 22207118336}
+{"step": 189, "generator_loss": 0.03355947509407997, "generator_grad_norm": 0.1325264275074005, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.913923263549805, "peak_memory_bytes": 22207118336}
+{"step": 190, "generator_loss": 0.07602763175964355, "generator_grad_norm": 0.1378926783800125, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9255576133728, "peak_memory_bytes": 22207118336}
+{"step": 191, "generator_loss": 0.030055636540055275, "generator_grad_norm": 0.07851491123437881, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.82487487792969, "peak_memory_bytes": 22207118336}
+{"step": 192, "generator_loss": 0.024286555126309395, "generator_grad_norm": 0.08335594832897186, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9850652217865, "peak_memory_bytes": 22207118336}
+{"step": 193, "generator_loss": 0.058485887944698334, "generator_grad_norm": 0.133774071931839, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.79729723930359, "peak_memory_bytes": 22207118336}
+{"step": 194, "generator_loss": 0.08737557381391525, "generator_grad_norm": 0.11254503577947617, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.947904109954834, "peak_memory_bytes": 22207118336}
+{"step": 195, "generator_loss": 0.061104338616132736, "generator_grad_norm": 0.11506140977144241, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.936206579208374, "peak_memory_bytes": 22207118336}
+{"step": 196, "generator_loss": 0.048169635236263275, "generator_grad_norm": 0.10343914479017258, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.830044746398926, "peak_memory_bytes": 22207118336}
+{"step": 197, "generator_loss": 0.08123684674501419, "generator_grad_norm": 0.11085242033004761, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.96055269241333, "peak_memory_bytes": 22207118336}
+{"step": 198, "generator_loss": 0.12577854096889496, "generator_grad_norm": 0.14499275386333466, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.775633573532104, "peak_memory_bytes": 22207118336}
+{"step": 199, "generator_loss": 0.02403096668422222, "generator_grad_norm": 0.1243942528963089, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.90405988693237, "peak_memory_bytes": 22207118336}
+{"step": 200, "generator_loss": 0.03727135807275772, "generator_grad_norm": 0.10443528741598129, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.884170055389404, "peak_memory_bytes": 22207118336}

outputs/temporal_nope/a1_rope_cpt_lr1e7_probe200_seed20260805_kerrigan_rerun1/validation_metrics.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "validation_loss": 0.06264185276813805, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 24.101814031600952, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 100, "validation_loss": 0.054266916238702834, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 23.307873010635376, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 200, "validation_loss": 0.05151267722249031, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 23.28480625152588, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}

outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt", "sha256": "ecc3f2d0dd47d812ba0cc58c252f4a75bf8051ea12bf1be3d2439d5cfe53b675", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt", "sha256": "24fb68a15b85bdd686990af64763ac2a23430f080979667a059c39c4aca083b3", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/resolved_config.yaml

@@ -0,0 +1,123 @@
+experiment:
+  id: a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: RoPE-CPT matched control
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 1.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+  target_visible_lengths:
+  - 21
+  - 42
+  - 84
+  train_length: 21
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 2.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/train.log

@@ -0,0 +1,259 @@
+W0806 17:49:07.823000 2947154 site-packages/torch/distributed/run.py:793] 
+W0806 17:49:07.823000 2947154 site-packages/torch/distributed/run.py:793] *****************************************
+W0806 17:49:07.823000 2947154 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0806 17:49:07.823000 2947154 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank1]:[W806 17:49:18.955023071 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank2]:[W806 17:49:18.132049069 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W806 17:49:18.161570839 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank4]:[W806 17:49:18.164929478 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank6]:[W806 17:49:18.189206043 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W806 17:49:18.192128904 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank7]:[W806 17:49:18.215928452 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank0]:[W806 17:49:18.420630722 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.06264185 alpha=1.000000 samples=32 seconds=22.796
+step=1 loss=0.16898617 grad_norm=0.284470 alpha=1.000000 seconds=33.868
+step=2 loss=0.08513896 grad_norm=0.261119 alpha=1.000000 seconds=33.740
+step=3 loss=0.07959563 grad_norm=0.312293 alpha=1.000000 seconds=34.054
+step=4 loss=0.11443653 grad_norm=0.284971 alpha=1.000000 seconds=34.042
+step=5 loss=0.04064651 grad_norm=0.425774 alpha=1.000000 seconds=34.140
+step=6 loss=0.04976147 grad_norm=0.236384 alpha=1.000000 seconds=34.075
+step=7 loss=0.12062521 grad_norm=0.284070 alpha=1.000000 seconds=34.156
+step=8 loss=0.04082027 grad_norm=0.261511 alpha=1.000000 seconds=34.211
+step=9 loss=0.03962434 grad_norm=0.265662 alpha=1.000000 seconds=34.057
+step=10 loss=0.11615936 grad_norm=0.335980 alpha=1.000000 seconds=34.225
+step=11 loss=0.05305076 grad_norm=0.281693 alpha=1.000000 seconds=33.985
+step=12 loss=0.03970905 grad_norm=0.200663 alpha=1.000000 seconds=34.176
+step=13 loss=0.06967629 grad_norm=0.229631 alpha=1.000000 seconds=34.170
+step=14 loss=0.11563975 grad_norm=0.241843 alpha=1.000000 seconds=34.034
+step=15 loss=0.02488876 grad_norm=0.250904 alpha=1.000000 seconds=34.192
+step=16 loss=0.07521610 grad_norm=0.237391 alpha=1.000000 seconds=34.006
+step=17 loss=0.16371337 grad_norm=0.270859 alpha=1.000000 seconds=34.150
+step=18 loss=0.05890699 grad_norm=0.220638 alpha=1.000000 seconds=33.981
+step=19 loss=0.05286266 grad_norm=0.277821 alpha=1.000000 seconds=34.146
+step=20 loss=0.07062636 grad_norm=0.185382 alpha=1.000000 seconds=34.175
+step=21 loss=0.03902369 grad_norm=0.240528 alpha=1.000000 seconds=34.089
+step=22 loss=0.10551379 grad_norm=0.245557 alpha=1.000000 seconds=34.162
+step=23 loss=0.05413212 grad_norm=0.198393 alpha=1.000000 seconds=34.126
+step=24 loss=0.03406988 grad_norm=0.212565 alpha=1.000000 seconds=33.970
+step=25 loss=0.03315390 grad_norm=0.219684 alpha=1.000000 seconds=34.106
+step=26 loss=0.05008332 grad_norm=0.181928 alpha=1.000000 seconds=34.008
+step=27 loss=0.10540730 grad_norm=0.187404 alpha=1.000000 seconds=34.127
+step=28 loss=0.04535390 grad_norm=0.165760 alpha=1.000000 seconds=34.026
+step=29 loss=0.09677438 grad_norm=0.168555 alpha=1.000000 seconds=34.147
+step=30 loss=0.04175425 grad_norm=0.234756 alpha=1.000000 seconds=34.236
+step=31 loss=0.12048307 grad_norm=0.235767 alpha=1.000000 seconds=34.018
+step=32 loss=0.02301337 grad_norm=0.185881 alpha=1.000000 seconds=34.201
+step=33 loss=0.02989142 grad_norm=0.247023 alpha=1.000000 seconds=34.216
+step=34 loss=0.08201148 grad_norm=0.169942 alpha=1.000000 seconds=34.050
+step=35 loss=0.04240957 grad_norm=0.144560 alpha=1.000000 seconds=34.242
+step=36 loss=0.08592667 grad_norm=0.206923 alpha=1.000000 seconds=34.135
+step=37 loss=0.07210538 grad_norm=0.186482 alpha=1.000000 seconds=34.296
+step=38 loss=0.05788248 grad_norm=0.183008 alpha=1.000000 seconds=34.090
+step=39 loss=0.03015511 grad_norm=0.152217 alpha=1.000000 seconds=34.166
+step=40 loss=0.03246506 grad_norm=0.145160 alpha=1.000000 seconds=34.192
+step=41 loss=0.06420499 grad_norm=0.226425 alpha=1.000000 seconds=34.177
+step=42 loss=0.04655270 grad_norm=0.131999 alpha=1.000000 seconds=34.209
+step=43 loss=0.01990617 grad_norm=0.244510 alpha=1.000000 seconds=34.240
+step=44 loss=0.05141648 grad_norm=0.141007 alpha=1.000000 seconds=34.040
+step=45 loss=0.04118113 grad_norm=0.148716 alpha=1.000000 seconds=34.232
+step=46 loss=0.04241344 grad_norm=0.173303 alpha=1.000000 seconds=34.011
+step=47 loss=0.03926999 grad_norm=0.182778 alpha=1.000000 seconds=34.181
+step=48 loss=0.07014707 grad_norm=0.145330 alpha=1.000000 seconds=34.024
+step=49 loss=0.03689032 grad_norm=0.135439 alpha=1.000000 seconds=34.182
+step=50 loss=0.07224321 grad_norm=0.165224 alpha=1.000000 seconds=34.143
+step=51 loss=0.03165115 grad_norm=0.180082 alpha=1.000000 seconds=34.048
+step=52 loss=0.02887170 grad_norm=0.194837 alpha=1.000000 seconds=34.176
+step=53 loss=0.04691007 grad_norm=0.146256 alpha=1.000000 seconds=34.142
+step=54 loss=0.02973087 grad_norm=0.125731 alpha=1.000000 seconds=34.040
+step=55 loss=0.07523579 grad_norm=0.128806 alpha=1.000000 seconds=34.200
+step=56 loss=0.07437647 grad_norm=0.123668 alpha=1.000000 seconds=34.023
+step=57 loss=0.03747848 grad_norm=0.127827 alpha=1.000000 seconds=34.164
+step=58 loss=0.03470153 grad_norm=0.105943 alpha=1.000000 seconds=34.029
+step=59 loss=0.11809643 grad_norm=0.147463 alpha=1.000000 seconds=34.168
+step=60 loss=0.08205218 grad_norm=0.119639 alpha=1.000000 seconds=34.189
+step=61 loss=0.06965017 grad_norm=0.166049 alpha=1.000000 seconds=34.064
+step=62 loss=0.10404476 grad_norm=0.156514 alpha=1.000000 seconds=34.208
+step=63 loss=0.07342021 grad_norm=0.123374 alpha=1.000000 seconds=34.288
+step=64 loss=0.05690050 grad_norm=0.108135 alpha=1.000000 seconds=34.110
+step=65 loss=0.02280051 grad_norm=0.119831 alpha=1.000000 seconds=34.263
+step=66 loss=0.02672077 grad_norm=0.126622 alpha=1.000000 seconds=34.117
+step=67 loss=0.01463608 grad_norm=0.092858 alpha=1.000000 seconds=34.278
+step=68 loss=0.10813227 grad_norm=0.127294 alpha=1.000000 seconds=34.075
+step=69 loss=0.06885134 grad_norm=0.175253 alpha=1.000000 seconds=34.150
+step=70 loss=0.03831206 grad_norm=0.125926 alpha=1.000000 seconds=34.166
+step=71 loss=0.03512514 grad_norm=0.125356 alpha=1.000000 seconds=34.014
+step=72 loss=0.04217657 grad_norm=0.121323 alpha=1.000000 seconds=34.108
+step=73 loss=0.16955227 grad_norm=0.190743 alpha=1.000000 seconds=34.127
+step=74 loss=0.07524139 grad_norm=0.136067 alpha=1.000000 seconds=33.955
+step=75 loss=0.09266371 grad_norm=0.108332 alpha=1.000000 seconds=34.196
+step=76 loss=0.02572753 grad_norm=0.119916 alpha=1.000000 seconds=33.973
+step=77 loss=0.04444459 grad_norm=0.114136 alpha=1.000000 seconds=34.128
+step=78 loss=0.07466266 grad_norm=0.099356 alpha=1.000000 seconds=33.944
+step=79 loss=0.04883490 grad_norm=0.127264 alpha=1.000000 seconds=34.086
+step=80 loss=0.05724190 grad_norm=0.144419 alpha=1.000000 seconds=34.056
+step=81 loss=0.05025553 grad_norm=0.072601 alpha=1.000000 seconds=33.994
+step=82 loss=0.03589023 grad_norm=0.101742 alpha=1.000000 seconds=34.119
+step=83 loss=0.03847847 grad_norm=0.101503 alpha=1.000000 seconds=34.102
+step=84 loss=0.12212365 grad_norm=0.109178 alpha=1.000000 seconds=33.946
+step=85 loss=0.02872832 grad_norm=0.136833 alpha=1.000000 seconds=34.063
+step=86 loss=0.02311257 grad_norm=0.099010 alpha=1.000000 seconds=33.906
+step=87 loss=0.03565622 grad_norm=0.115263 alpha=1.000000 seconds=34.115
+step=88 loss=0.06111155 grad_norm=0.123879 alpha=1.000000 seconds=33.951
+step=89 loss=0.13542603 grad_norm=0.113852 alpha=1.000000 seconds=34.081
+step=90 loss=0.09270129 grad_norm=0.115964 alpha=1.000000 seconds=34.137
+step=91 loss=0.04497743 grad_norm=0.110884 alpha=1.000000 seconds=33.975
+step=92 loss=0.02086187 grad_norm=0.103153 alpha=1.000000 seconds=34.114
+step=93 loss=0.02145690 grad_norm=0.105451 alpha=1.000000 seconds=34.108
+step=94 loss=0.12779032 grad_norm=0.105955 alpha=1.000000 seconds=33.926
+step=95 loss=0.02601034 grad_norm=0.092761 alpha=1.000000 seconds=34.058
+step=96 loss=0.17284752 grad_norm=0.115497 alpha=1.000000 seconds=33.951
+step=97 loss=0.12222225 grad_norm=0.112921 alpha=1.000000 seconds=34.111
+step=98 loss=0.07713963 grad_norm=0.153858 alpha=1.000000 seconds=33.944
+step=99 loss=0.03956973 grad_norm=0.107386 alpha=1.000000 seconds=34.075
+step=100 loss=0.06718601 grad_norm=0.084812 alpha=1.000000 seconds=34.071
+validation step=100 loss=0.05149416 alpha=1.000000 samples=32 seconds=23.614
+Start gathering distributed model states...
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+Start gathering distributed model states...Start gathering distributed model states...
+
+Start gathering distributed model states...Start gathering distributed model states.../home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+
+
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt sha256=ecc3f2d0dd47d812ba0cc58c252f4a75bf8051ea12bf1be3d2439d5cfe53b675
+step=101 loss=0.06273130 grad_norm=0.111810 alpha=1.000000 seconds=33.361
+step=102 loss=0.03461096 grad_norm=0.060189 alpha=1.000000 seconds=34.063
+step=103 loss=0.06924274 grad_norm=0.073391 alpha=1.000000 seconds=33.967
+step=104 loss=0.16222589 grad_norm=0.130340 alpha=1.000000 seconds=34.070
+step=105 loss=0.02396709 grad_norm=0.069984 alpha=1.000000 seconds=33.944
+step=106 loss=0.02275083 grad_norm=0.132807 alpha=1.000000 seconds=34.117
+step=107 loss=0.03428217 grad_norm=0.095892 alpha=1.000000 seconds=34.130
+step=108 loss=0.06227133 grad_norm=0.104845 alpha=1.000000 seconds=33.965
+step=109 loss=0.20930013 grad_norm=0.101076 alpha=1.000000 seconds=34.066
+step=110 loss=0.03029286 grad_norm=0.099945 alpha=1.000000 seconds=33.940
+step=111 loss=0.07052343 grad_norm=0.086051 alpha=1.000000 seconds=34.075
+step=112 loss=0.13282412 grad_norm=0.106937 alpha=1.000000 seconds=34.074
+step=113 loss=0.12274679 grad_norm=0.107786 alpha=1.000000 seconds=33.920
+step=114 loss=0.03393879 grad_norm=0.087109 alpha=1.000000 seconds=34.018
+step=115 loss=0.06133206 grad_norm=0.116902 alpha=1.000000 seconds=33.960
+step=116 loss=0.03052677 grad_norm=0.083880 alpha=1.000000 seconds=34.036
+step=117 loss=0.09849478 grad_norm=0.097296 alpha=1.000000 seconds=33.914
+step=118 loss=0.02344885 grad_norm=0.113836 alpha=1.000000 seconds=34.078
+step=119 loss=0.05764988 grad_norm=0.088984 alpha=1.000000 seconds=34.083
+step=120 loss=0.05431110 grad_norm=0.089279 alpha=1.000000 seconds=33.940
+step=121 loss=0.03885333 grad_norm=0.086932 alpha=1.000000 seconds=34.080
+step=122 loss=0.02417460 grad_norm=0.098609 alpha=1.000000 seconds=33.914
+step=123 loss=0.06613803 grad_norm=0.076987 alpha=1.000000 seconds=34.072
+step=124 loss=0.02609912 grad_norm=0.082354 alpha=1.000000 seconds=33.932
+step=125 loss=0.09971292 grad_norm=0.113892 alpha=1.000000 seconds=34.061
+step=126 loss=0.16203056 grad_norm=0.088892 alpha=1.000000 seconds=34.062
+step=127 loss=0.06556828 grad_norm=0.072106 alpha=1.000000 seconds=33.951
+step=128 loss=0.03675062 grad_norm=0.074732 alpha=1.000000 seconds=34.057
+step=129 loss=0.06720750 grad_norm=0.099632 alpha=1.000000 seconds=33.941
+step=130 loss=0.05066364 grad_norm=0.082995 alpha=1.000000 seconds=34.085
+step=131 loss=0.06381852 grad_norm=0.078325 alpha=1.000000 seconds=33.921
+step=132 loss=0.03607829 grad_norm=0.085399 alpha=1.000000 seconds=34.018
+step=133 loss=0.09722326 grad_norm=0.096649 alpha=1.000000 seconds=33.894
+step=134 loss=0.09013963 grad_norm=0.100355 alpha=1.000000 seconds=34.029
+step=135 loss=0.03181169 grad_norm=0.073833 alpha=1.000000 seconds=34.058
+step=136 loss=0.04963914 grad_norm=0.073793 alpha=1.000000 seconds=33.941
+step=137 loss=0.09497651 grad_norm=0.109182 alpha=1.000000 seconds=34.015
+step=138 loss=0.12135700 grad_norm=0.081676 alpha=1.000000 seconds=33.965
+step=139 loss=0.03926191 grad_norm=0.078868 alpha=1.000000 seconds=34.061
+step=140 loss=0.01565489 grad_norm=0.108443 alpha=1.000000 seconds=34.166
+step=141 loss=0.04840000 grad_norm=0.080068 alpha=1.000000 seconds=33.978
+step=142 loss=0.09986068 grad_norm=0.090320 alpha=1.000000 seconds=34.060
+step=143 loss=0.03111730 grad_norm=0.115566 alpha=1.000000 seconds=34.039
+step=144 loss=0.02470786 grad_norm=0.106382 alpha=1.000000 seconds=34.056
+step=145 loss=0.07313292 grad_norm=0.063001 alpha=1.000000 seconds=33.957
+step=146 loss=0.01989143 grad_norm=0.142112 alpha=1.000000 seconds=34.083
+step=147 loss=0.05152301 grad_norm=0.069109 alpha=1.000000 seconds=34.017
+step=148 loss=0.07280886 grad_norm=0.065295 alpha=1.000000 seconds=33.937
+step=149 loss=0.03424121 grad_norm=0.100068 alpha=1.000000 seconds=34.065
+step=150 loss=0.06957927 grad_norm=0.084937 alpha=1.000000 seconds=33.966
+step=151 loss=0.03405398 grad_norm=0.071146 alpha=1.000000 seconds=34.059
+step=152 loss=0.14615726 grad_norm=0.077616 alpha=1.000000 seconds=33.965
+step=153 loss=0.03208911 grad_norm=0.054108 alpha=1.000000 seconds=34.031
+step=154 loss=0.01547801 grad_norm=0.065531 alpha=1.000000 seconds=34.059
+step=155 loss=0.03527249 grad_norm=0.059721 alpha=1.000000 seconds=33.883
+step=156 loss=0.03440324 grad_norm=0.081641 alpha=1.000000 seconds=34.028
+step=157 loss=0.03447662 grad_norm=0.066896 alpha=1.000000 seconds=33.901
+step=158 loss=0.05317080 grad_norm=0.079332 alpha=1.000000 seconds=34.043
+step=159 loss=0.02017602 grad_norm=0.055633 alpha=1.000000 seconds=33.938
+step=160 loss=0.02443529 grad_norm=0.062283 alpha=1.000000 seconds=34.000
+step=161 loss=0.04842632 grad_norm=0.083819 alpha=1.000000 seconds=34.113
+step=162 loss=0.06595819 grad_norm=0.059462 alpha=1.000000 seconds=33.913
+step=163 loss=0.03688005 grad_norm=0.054808 alpha=1.000000 seconds=34.148
+step=164 loss=0.02285131 grad_norm=0.063991 alpha=1.000000 seconds=33.915
+step=165 loss=0.05659306 grad_norm=0.079114 alpha=1.000000 seconds=34.011
+step=166 loss=0.02821723 grad_norm=0.090821 alpha=1.000000 seconds=33.903
+step=167 loss=0.03798281 grad_norm=0.108425 alpha=1.000000 seconds=34.077
+step=168 loss=0.01567173 grad_norm=0.093247 alpha=1.000000 seconds=34.061
+step=169 loss=0.09173944 grad_norm=0.064180 alpha=1.000000 seconds=33.897
+step=170 loss=0.06947253 grad_norm=0.065640 alpha=1.000000 seconds=34.004
+step=171 loss=0.04222508 grad_norm=0.053593 alpha=1.000000 seconds=33.902
+step=172 loss=0.02352354 grad_norm=0.063409 alpha=1.000000 seconds=34.008
+step=173 loss=0.11869207 grad_norm=0.079329 alpha=1.000000 seconds=33.949
+step=174 loss=0.03655270 grad_norm=0.063556 alpha=1.000000 seconds=34.108
+step=175 loss=0.09101053 grad_norm=0.080198 alpha=1.000000 seconds=33.997
+step=176 loss=0.04128627 grad_norm=0.069390 alpha=1.000000 seconds=33.980
+step=177 loss=0.07822479 grad_norm=0.092013 alpha=1.000000 seconds=34.034
+step=178 loss=0.13857360 grad_norm=0.089565 alpha=1.000000 seconds=33.888
+step=179 loss=0.04407916 grad_norm=0.056674 alpha=1.000000 seconds=34.033
+step=180 loss=0.03893535 grad_norm=0.071359 alpha=1.000000 seconds=34.038
+step=181 loss=0.02897717 grad_norm=0.061748 alpha=1.000000 seconds=33.891
+step=182 loss=0.10060429 grad_norm=0.078757 alpha=1.000000 seconds=34.081
+step=183 loss=0.02858094 grad_norm=0.053235 alpha=1.000000 seconds=33.935
+step=184 loss=0.03874340 grad_norm=0.073906 alpha=1.000000 seconds=34.019
+step=185 loss=0.05365682 grad_norm=0.076958 alpha=1.000000 seconds=33.935
+step=186 loss=0.04486073 grad_norm=0.068309 alpha=1.000000 seconds=34.027
+step=187 loss=0.03903292 grad_norm=0.056905 alpha=1.000000 seconds=34.036
+step=188 loss=0.05954648 grad_norm=0.065843 alpha=1.000000 seconds=33.906
+step=189 loss=0.03222750 grad_norm=0.075989 alpha=1.000000 seconds=34.033
+step=190 loss=0.07377744 grad_norm=0.094123 alpha=1.000000 seconds=33.890
+step=191 loss=0.02874019 grad_norm=0.049843 alpha=1.000000 seconds=34.085
+step=192 loss=0.02298055 grad_norm=0.053336 alpha=1.000000 seconds=34.055
+step=193 loss=0.05588349 grad_norm=0.087187 alpha=1.000000 seconds=33.912
+step=194 loss=0.08506828 grad_norm=0.085927 alpha=1.000000 seconds=34.035
+step=195 loss=0.05807612 grad_norm=0.079070 alpha=1.000000 seconds=33.961
+step=196 loss=0.04578342 grad_norm=0.076502 alpha=1.000000 seconds=34.038
+step=197 loss=0.07894004 grad_norm=0.084067 alpha=1.000000 seconds=33.888
+step=198 loss=0.12087275 grad_norm=0.116431 alpha=1.000000 seconds=34.046
+step=199 loss=0.02331393 grad_norm=0.089294 alpha=1.000000 seconds=34.033
+step=200 loss=0.03558746 grad_norm=0.075185 alpha=1.000000 seconds=33.902
+validation step=200 loss=0.04925254 alpha=1.000000 samples=32 seconds=23.490
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt sha256=24fb68a15b85bdd686990af64763ac2a23430f080979667a059c39c4aca083b3

outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/training_metrics.jsonl

@@ -0,0 +1,200 @@
+{"step": 1, "generator_loss": 0.16898617148399353, "generator_grad_norm": 0.28447043895721436, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.867755651474, "peak_memory_bytes": 20785906176}
+{"step": 2, "generator_loss": 0.0851389616727829, "generator_grad_norm": 0.26111921668052673, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.73996019363403, "peak_memory_bytes": 22206258176}
+{"step": 3, "generator_loss": 0.07959563285112381, "generator_grad_norm": 0.31229281425476074, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05399131774902, "peak_memory_bytes": 22206258176}
+{"step": 4, "generator_loss": 0.11443652957677841, "generator_grad_norm": 0.28497081995010376, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.04233765602112, "peak_memory_bytes": 22206258176}
+{"step": 5, "generator_loss": 0.0406465083360672, "generator_grad_norm": 0.42577382922172546, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.1402051448822, "peak_memory_bytes": 22206694400}
+{"step": 6, "generator_loss": 0.049761466681957245, "generator_grad_norm": 0.23638387024402618, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07512307167053, "peak_memory_bytes": 22206694400}
+{"step": 7, "generator_loss": 0.12062520533800125, "generator_grad_norm": 0.2840704023838043, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.156049489974976, "peak_memory_bytes": 22206694400}
+{"step": 8, "generator_loss": 0.040820274502038956, "generator_grad_norm": 0.2615108788013458, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.211313009262085, "peak_memory_bytes": 22206694400}
+{"step": 9, "generator_loss": 0.03962433710694313, "generator_grad_norm": 0.2656620442867279, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05704116821289, "peak_memory_bytes": 22206694400}
+{"step": 10, "generator_loss": 0.1161593645811081, "generator_grad_norm": 0.33598002791404724, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.225322246551514, "peak_memory_bytes": 22206694400}
+{"step": 11, "generator_loss": 0.05305075645446777, "generator_grad_norm": 0.28169310092926025, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.985074281692505, "peak_memory_bytes": 22206694400}
+{"step": 12, "generator_loss": 0.03970905393362045, "generator_grad_norm": 0.20066344738006592, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.175987243652344, "peak_memory_bytes": 22206694400}
+{"step": 13, "generator_loss": 0.06967629492282867, "generator_grad_norm": 0.22963108122348785, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.17007374763489, "peak_memory_bytes": 22206694400}
+{"step": 14, "generator_loss": 0.11563975363969803, "generator_grad_norm": 0.24184349179267883, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.034419536590576, "peak_memory_bytes": 22206694400}
+{"step": 15, "generator_loss": 0.02488875947892666, "generator_grad_norm": 0.25090447068214417, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.192402839660645, "peak_memory_bytes": 22206694400}
+{"step": 16, "generator_loss": 0.07521609961986542, "generator_grad_norm": 0.23739075660705566, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.005605936050415, "peak_memory_bytes": 22206694400}
+{"step": 17, "generator_loss": 0.16371336579322815, "generator_grad_norm": 0.2708592712879181, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.15022921562195, "peak_memory_bytes": 22206694400}
+{"step": 18, "generator_loss": 0.05890698730945587, "generator_grad_norm": 0.22063793241977692, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.98147630691528, "peak_memory_bytes": 22206694400}
+{"step": 19, "generator_loss": 0.052862655371427536, "generator_grad_norm": 0.27782148122787476, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.1456241607666, "peak_memory_bytes": 22206694400}
+{"step": 20, "generator_loss": 0.07062635570764542, "generator_grad_norm": 0.18538235127925873, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.174606800079346, "peak_memory_bytes": 22206831616}
+{"step": 21, "generator_loss": 0.03902369365096092, "generator_grad_norm": 0.2405279129743576, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08908295631409, "peak_memory_bytes": 22206831616}
+{"step": 22, "generator_loss": 0.1055137887597084, "generator_grad_norm": 0.24555695056915283, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.16169714927673, "peak_memory_bytes": 22206831616}
+{"step": 23, "generator_loss": 0.0541321225464344, "generator_grad_norm": 0.19839346408843994, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.125553131103516, "peak_memory_bytes": 22206981120}
+{"step": 24, "generator_loss": 0.034069884568452835, "generator_grad_norm": 0.212564617395401, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.97027015686035, "peak_memory_bytes": 22206981120}
+{"step": 25, "generator_loss": 0.033153899013996124, "generator_grad_norm": 0.21968387067317963, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.10603737831116, "peak_memory_bytes": 22206981120}
+{"step": 26, "generator_loss": 0.05008332431316376, "generator_grad_norm": 0.18192769587039948, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.00758194923401, "peak_memory_bytes": 22206981120}
+{"step": 27, "generator_loss": 0.10540729761123657, "generator_grad_norm": 0.18740440905094147, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.127379417419434, "peak_memory_bytes": 22206981120}
+{"step": 28, "generator_loss": 0.04535389691591263, "generator_grad_norm": 0.16575990617275238, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.025968074798584, "peak_memory_bytes": 22206981120}
+{"step": 29, "generator_loss": 0.0967743769288063, "generator_grad_norm": 0.1685548722743988, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.14713215827942, "peak_memory_bytes": 22206981120}
+{"step": 30, "generator_loss": 0.04175424575805664, "generator_grad_norm": 0.23475594818592072, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.23570728302002, "peak_memory_bytes": 22206981120}
+{"step": 31, "generator_loss": 0.12048307061195374, "generator_grad_norm": 0.23576657474040985, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01827883720398, "peak_memory_bytes": 22206981120}
+{"step": 32, "generator_loss": 0.023013373836874962, "generator_grad_norm": 0.18588115274906158, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.20066690444946, "peak_memory_bytes": 22206981120}
+{"step": 33, "generator_loss": 0.029891423881053925, "generator_grad_norm": 0.24702313542366028, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.215516328811646, "peak_memory_bytes": 22206981120}
+{"step": 34, "generator_loss": 0.08201147615909576, "generator_grad_norm": 0.16994211077690125, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05040264129639, "peak_memory_bytes": 22206981120}
+{"step": 35, "generator_loss": 0.04240957275032997, "generator_grad_norm": 0.144560307264328, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.24211645126343, "peak_memory_bytes": 22206981120}
+{"step": 36, "generator_loss": 0.08592667430639267, "generator_grad_norm": 0.20692312717437744, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.13493800163269, "peak_memory_bytes": 22206981120}
+{"step": 37, "generator_loss": 0.07210537791252136, "generator_grad_norm": 0.1864815652370453, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.295981645584106, "peak_memory_bytes": 22206981120}
+{"step": 38, "generator_loss": 0.05788248032331467, "generator_grad_norm": 0.1830080896615982, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.09009671211243, "peak_memory_bytes": 22206981120}
+{"step": 39, "generator_loss": 0.030155105516314507, "generator_grad_norm": 0.15221723914146423, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.16574287414551, "peak_memory_bytes": 22206981120}
+{"step": 40, "generator_loss": 0.03246505931019783, "generator_grad_norm": 0.14515992999076843, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.19167470932007, "peak_memory_bytes": 22206981120}
+{"step": 41, "generator_loss": 0.06420499086380005, "generator_grad_norm": 0.22642479836940765, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.17679500579834, "peak_memory_bytes": 22206981120}
+{"step": 42, "generator_loss": 0.04655270278453827, "generator_grad_norm": 0.13199853897094727, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.20944786071777, "peak_memory_bytes": 22206981120}
+{"step": 43, "generator_loss": 0.019906166940927505, "generator_grad_norm": 0.24451018869876862, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.24002432823181, "peak_memory_bytes": 22206981120}
+{"step": 44, "generator_loss": 0.05141647905111313, "generator_grad_norm": 0.14100675284862518, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.0395987033844, "peak_memory_bytes": 22206981120}
+{"step": 45, "generator_loss": 0.041181132197380066, "generator_grad_norm": 0.1487155258655548, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.23185729980469, "peak_memory_bytes": 22206981120}
+{"step": 46, "generator_loss": 0.042413439601659775, "generator_grad_norm": 0.17330342531204224, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01114320755005, "peak_memory_bytes": 22206981120}
+{"step": 47, "generator_loss": 0.03926999494433403, "generator_grad_norm": 0.18277780711650848, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.18088626861572, "peak_memory_bytes": 22207118336}
+{"step": 48, "generator_loss": 0.0701470747590065, "generator_grad_norm": 0.14533016085624695, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.024091958999634, "peak_memory_bytes": 22207118336}
+{"step": 49, "generator_loss": 0.03689032047986984, "generator_grad_norm": 0.13543851673603058, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.18159604072571, "peak_memory_bytes": 22207118336}
+{"step": 50, "generator_loss": 0.07224320620298386, "generator_grad_norm": 0.16522379219532013, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.14264917373657, "peak_memory_bytes": 22207118336}
+{"step": 51, "generator_loss": 0.03165115416049957, "generator_grad_norm": 0.1800820380449295, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.04843235015869, "peak_memory_bytes": 22207118336}
+{"step": 52, "generator_loss": 0.028871702030301094, "generator_grad_norm": 0.19483749568462372, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.17648100852966, "peak_memory_bytes": 22207118336}
+{"step": 53, "generator_loss": 0.04691006615757942, "generator_grad_norm": 0.14625631272792816, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.14229941368103, "peak_memory_bytes": 22207118336}
+{"step": 54, "generator_loss": 0.029730871319770813, "generator_grad_norm": 0.12573090195655823, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.039690256118774, "peak_memory_bytes": 22207118336}
+{"step": 55, "generator_loss": 0.07523579150438309, "generator_grad_norm": 0.1288057565689087, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.199604749679565, "peak_memory_bytes": 22207118336}
+{"step": 56, "generator_loss": 0.07437647134065628, "generator_grad_norm": 0.12366797029972076, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.02285981178284, "peak_memory_bytes": 22207118336}
+{"step": 57, "generator_loss": 0.037478476762771606, "generator_grad_norm": 0.12782740592956543, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.163816690444946, "peak_memory_bytes": 22207118336}
+{"step": 58, "generator_loss": 0.03470153361558914, "generator_grad_norm": 0.10594271123409271, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.029104232788086, "peak_memory_bytes": 22207118336}
+{"step": 59, "generator_loss": 0.11809643357992172, "generator_grad_norm": 0.14746250212192535, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.167633056640625, "peak_memory_bytes": 22207118336}
+{"step": 60, "generator_loss": 0.08205217868089676, "generator_grad_norm": 0.11963909864425659, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.18880248069763, "peak_memory_bytes": 22207118336}
+{"step": 61, "generator_loss": 0.06965016573667526, "generator_grad_norm": 0.16604933142662048, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06436824798584, "peak_memory_bytes": 22207118336}
+{"step": 62, "generator_loss": 0.10404475778341293, "generator_grad_norm": 0.1565137505531311, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.20754837989807, "peak_memory_bytes": 22207118336}
+{"step": 63, "generator_loss": 0.0734202116727829, "generator_grad_norm": 0.12337438017129898, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.28843021392822, "peak_memory_bytes": 22207118336}
+{"step": 64, "generator_loss": 0.0569005012512207, "generator_grad_norm": 0.10813505947589874, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.10975217819214, "peak_memory_bytes": 22207118336}
+{"step": 65, "generator_loss": 0.022800510749220848, "generator_grad_norm": 0.11983145028352737, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.26308870315552, "peak_memory_bytes": 22207118336}
+{"step": 66, "generator_loss": 0.026720773428678513, "generator_grad_norm": 0.12662188708782196, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.116652488708496, "peak_memory_bytes": 22207118336}
+{"step": 67, "generator_loss": 0.014636080712080002, "generator_grad_norm": 0.09285753220319748, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.277833223342896, "peak_memory_bytes": 22207118336}
+{"step": 68, "generator_loss": 0.1081322729587555, "generator_grad_norm": 0.127293661236763, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07549047470093, "peak_memory_bytes": 22207118336}
+{"step": 69, "generator_loss": 0.06885133683681488, "generator_grad_norm": 0.17525315284729004, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.15010643005371, "peak_memory_bytes": 22207118336}
+{"step": 70, "generator_loss": 0.03831206262111664, "generator_grad_norm": 0.12592639029026031, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.165995836257935, "peak_memory_bytes": 22207118336}
+{"step": 71, "generator_loss": 0.03512514382600784, "generator_grad_norm": 0.12535598874092102, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01391863822937, "peak_memory_bytes": 22207118336}
+{"step": 72, "generator_loss": 0.04217657074332237, "generator_grad_norm": 0.12132324278354645, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.108378887176514, "peak_memory_bytes": 22207118336}
+{"step": 73, "generator_loss": 0.1695522665977478, "generator_grad_norm": 0.19074319303035736, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.127259492874146, "peak_memory_bytes": 22207118336}
+{"step": 74, "generator_loss": 0.07524139434099197, "generator_grad_norm": 0.13606712222099304, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.95518755912781, "peak_memory_bytes": 22207118336}
+{"step": 75, "generator_loss": 0.0926637127995491, "generator_grad_norm": 0.1083323284983635, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.1959810256958, "peak_memory_bytes": 22207118336}
+{"step": 76, "generator_loss": 0.02572752721607685, "generator_grad_norm": 0.11991550028324127, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.97268867492676, "peak_memory_bytes": 22207118336}
+{"step": 77, "generator_loss": 0.04444459080696106, "generator_grad_norm": 0.11413571238517761, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.12827920913696, "peak_memory_bytes": 22207118336}
+{"step": 78, "generator_loss": 0.07466266304254532, "generator_grad_norm": 0.09935551136732101, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.944026470184326, "peak_memory_bytes": 22207118336}
+{"step": 79, "generator_loss": 0.048834897577762604, "generator_grad_norm": 0.1272636353969574, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08589243888855, "peak_memory_bytes": 22207118336}
+{"step": 80, "generator_loss": 0.05724190175533295, "generator_grad_norm": 0.1444188356399536, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05573129653931, "peak_memory_bytes": 22207118336}
+{"step": 81, "generator_loss": 0.05025552585721016, "generator_grad_norm": 0.0726006031036377, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.994192600250244, "peak_memory_bytes": 22207118336}
+{"step": 82, "generator_loss": 0.03589022904634476, "generator_grad_norm": 0.10174209624528885, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.119017362594604, "peak_memory_bytes": 22207118336}
+{"step": 83, "generator_loss": 0.03847847133874893, "generator_grad_norm": 0.10150302201509476, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.102315187454224, "peak_memory_bytes": 22207118336}
+{"step": 84, "generator_loss": 0.12212365120649338, "generator_grad_norm": 0.10917836427688599, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.94559168815613, "peak_memory_bytes": 22207118336}
+{"step": 85, "generator_loss": 0.028728323057293892, "generator_grad_norm": 0.13683302700519562, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.062782287597656, "peak_memory_bytes": 22207118336}
+{"step": 86, "generator_loss": 0.023112572729587555, "generator_grad_norm": 0.09901038557291031, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.906068086624146, "peak_memory_bytes": 22207118336}
+{"step": 87, "generator_loss": 0.035656217485666275, "generator_grad_norm": 0.11526311188936234, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.114766359329224, "peak_memory_bytes": 22207118336}
+{"step": 88, "generator_loss": 0.06111155450344086, "generator_grad_norm": 0.12387928366661072, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.95093822479248, "peak_memory_bytes": 22207118336}
+{"step": 89, "generator_loss": 0.13542602956295013, "generator_grad_norm": 0.11385223269462585, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.081297636032104, "peak_memory_bytes": 22207118336}
+{"step": 90, "generator_loss": 0.09270128607749939, "generator_grad_norm": 0.11596444994211197, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.13739728927612, "peak_memory_bytes": 22207118336}
+{"step": 91, "generator_loss": 0.04497743397951126, "generator_grad_norm": 0.11088357120752335, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.97466206550598, "peak_memory_bytes": 22207118336}
+{"step": 92, "generator_loss": 0.020861873403191566, "generator_grad_norm": 0.10315269231796265, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.11432385444641, "peak_memory_bytes": 22207118336}
+{"step": 93, "generator_loss": 0.021456902846693993, "generator_grad_norm": 0.10545149445533752, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.10780429840088, "peak_memory_bytes": 22207118336}
+{"step": 94, "generator_loss": 0.12779031693935394, "generator_grad_norm": 0.10595521330833435, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.92635989189148, "peak_memory_bytes": 22207118336}
+{"step": 95, "generator_loss": 0.026010343804955482, "generator_grad_norm": 0.09276139736175537, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05801296234131, "peak_memory_bytes": 22207118336}
+{"step": 96, "generator_loss": 0.17284752428531647, "generator_grad_norm": 0.11549685150384903, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.951491832733154, "peak_memory_bytes": 22207118336}
+{"step": 97, "generator_loss": 0.12222225219011307, "generator_grad_norm": 0.11292107403278351, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.11136317253113, "peak_memory_bytes": 22207118336}
+{"step": 98, "generator_loss": 0.07713963091373444, "generator_grad_norm": 0.1538582742214203, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.943801164627075, "peak_memory_bytes": 22207118336}
+{"step": 99, "generator_loss": 0.039569731801748276, "generator_grad_norm": 0.10738629847764969, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07513618469238, "peak_memory_bytes": 22207118336}
+{"step": 100, "generator_loss": 0.06718600541353226, "generator_grad_norm": 0.08481196314096451, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07114601135254, "peak_memory_bytes": 22207118336}
+{"step": 101, "generator_loss": 0.0627313032746315, "generator_grad_norm": 0.11181044578552246, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.36074924468994, "peak_memory_bytes": 22207118336}
+{"step": 102, "generator_loss": 0.03461096063256264, "generator_grad_norm": 0.060188762843608856, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.062814474105835, "peak_memory_bytes": 22207118336}
+{"step": 103, "generator_loss": 0.06924273818731308, "generator_grad_norm": 0.07339063286781311, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.96652412414551, "peak_memory_bytes": 22207118336}
+{"step": 104, "generator_loss": 0.1622258871793747, "generator_grad_norm": 0.1303398758172989, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.070239543914795, "peak_memory_bytes": 22207118336}
+{"step": 105, "generator_loss": 0.023967085406184196, "generator_grad_norm": 0.06998440623283386, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.94420528411865, "peak_memory_bytes": 22207118336}
+{"step": 106, "generator_loss": 0.02275083214044571, "generator_grad_norm": 0.13280685245990753, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.117356061935425, "peak_memory_bytes": 22207118336}
+{"step": 107, "generator_loss": 0.03428216651082039, "generator_grad_norm": 0.09589169174432755, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.130298376083374, "peak_memory_bytes": 22207118336}
+{"step": 108, "generator_loss": 0.06227133423089981, "generator_grad_norm": 0.10484546422958374, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.965067863464355, "peak_memory_bytes": 22207118336}
+{"step": 109, "generator_loss": 0.20930013060569763, "generator_grad_norm": 0.1010756567120552, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06646680831909, "peak_memory_bytes": 22207118336}
+{"step": 110, "generator_loss": 0.03029286116361618, "generator_grad_norm": 0.09994519501924515, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.940319299697876, "peak_memory_bytes": 22207118336}
+{"step": 111, "generator_loss": 0.07052342593669891, "generator_grad_norm": 0.086050845682621, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07487750053406, "peak_memory_bytes": 22207118336}
+{"step": 112, "generator_loss": 0.1328241229057312, "generator_grad_norm": 0.10693734139204025, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.074461698532104, "peak_memory_bytes": 22207118336}
+{"step": 113, "generator_loss": 0.1227467879652977, "generator_grad_norm": 0.10778612643480301, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9204158782959, "peak_memory_bytes": 22207118336}
+{"step": 114, "generator_loss": 0.03393878787755966, "generator_grad_norm": 0.08710937201976776, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01810002326965, "peak_memory_bytes": 22207118336}
+{"step": 115, "generator_loss": 0.061332058161497116, "generator_grad_norm": 0.1169017031788826, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.95951843261719, "peak_memory_bytes": 22207118336}
+{"step": 116, "generator_loss": 0.030526774004101753, "generator_grad_norm": 0.08388010412454605, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03577280044556, "peak_memory_bytes": 22207118336}
+{"step": 117, "generator_loss": 0.09849478304386139, "generator_grad_norm": 0.09729646146297455, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9142689704895, "peak_memory_bytes": 22207118336}
+{"step": 118, "generator_loss": 0.023448847234249115, "generator_grad_norm": 0.11383616179227829, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07822513580322, "peak_memory_bytes": 22207118336}
+{"step": 119, "generator_loss": 0.0576498806476593, "generator_grad_norm": 0.08898358792066574, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08298873901367, "peak_memory_bytes": 22207118336}
+{"step": 120, "generator_loss": 0.05431109666824341, "generator_grad_norm": 0.08927924931049347, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93953061103821, "peak_memory_bytes": 22207118336}
+{"step": 121, "generator_loss": 0.03885332867503166, "generator_grad_norm": 0.0869317427277565, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07981514930725, "peak_memory_bytes": 22207118336}
+{"step": 122, "generator_loss": 0.024174602702260017, "generator_grad_norm": 0.09860934317111969, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.91387701034546, "peak_memory_bytes": 22207118336}
+{"step": 123, "generator_loss": 0.06613802909851074, "generator_grad_norm": 0.0769868940114975, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.072351932525635, "peak_memory_bytes": 22207118336}
+{"step": 124, "generator_loss": 0.02609912119805813, "generator_grad_norm": 0.08235390484333038, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93230938911438, "peak_memory_bytes": 22207118336}
+{"step": 125, "generator_loss": 0.09971291571855545, "generator_grad_norm": 0.11389194428920746, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06105327606201, "peak_memory_bytes": 22207118336}
+{"step": 126, "generator_loss": 0.16203056275844574, "generator_grad_norm": 0.08889196068048477, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06156301498413, "peak_memory_bytes": 22207118336}
+{"step": 127, "generator_loss": 0.06556827574968338, "generator_grad_norm": 0.0721064954996109, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9507052898407, "peak_memory_bytes": 22207118336}
+{"step": 128, "generator_loss": 0.03675061836838722, "generator_grad_norm": 0.07473184168338776, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.057443618774414, "peak_memory_bytes": 22207118336}
+{"step": 129, "generator_loss": 0.06720750033855438, "generator_grad_norm": 0.09963224083185196, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9414701461792, "peak_memory_bytes": 22207118336}
+{"step": 130, "generator_loss": 0.05066364258527756, "generator_grad_norm": 0.08299455046653748, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08470010757446, "peak_memory_bytes": 22207118336}
+{"step": 131, "generator_loss": 0.06381852179765701, "generator_grad_norm": 0.07832544296979904, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.920963287353516, "peak_memory_bytes": 22207118336}
+{"step": 132, "generator_loss": 0.03607828915119171, "generator_grad_norm": 0.08539878576993942, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01797080039978, "peak_memory_bytes": 22207118336}
+{"step": 133, "generator_loss": 0.09722325950860977, "generator_grad_norm": 0.09664921462535858, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8941867351532, "peak_memory_bytes": 22207118336}
+{"step": 134, "generator_loss": 0.09013962745666504, "generator_grad_norm": 0.10035530477762222, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.02943444252014, "peak_memory_bytes": 22207118336}
+{"step": 135, "generator_loss": 0.03181169182062149, "generator_grad_norm": 0.07383289933204651, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05843162536621, "peak_memory_bytes": 22207118336}
+{"step": 136, "generator_loss": 0.04963913932442665, "generator_grad_norm": 0.0737927109003067, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.94082736968994, "peak_memory_bytes": 22207118336}
+{"step": 137, "generator_loss": 0.094976507127285, "generator_grad_norm": 0.10918190330266953, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01513075828552, "peak_memory_bytes": 22207118336}
+{"step": 138, "generator_loss": 0.12135700136423111, "generator_grad_norm": 0.08167552202939987, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.96515226364136, "peak_memory_bytes": 22207118336}
+{"step": 139, "generator_loss": 0.03926191106438637, "generator_grad_norm": 0.07886835187673569, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.061179876327515, "peak_memory_bytes": 22207118336}
+{"step": 140, "generator_loss": 0.01565488614141941, "generator_grad_norm": 0.10844335705041885, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.166239500045776, "peak_memory_bytes": 22207118336}
+{"step": 141, "generator_loss": 0.04839999973773956, "generator_grad_norm": 0.08006816357374191, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.97802805900574, "peak_memory_bytes": 22207118336}
+{"step": 142, "generator_loss": 0.09986068308353424, "generator_grad_norm": 0.09032013267278671, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06035232543945, "peak_memory_bytes": 22207118336}
+{"step": 143, "generator_loss": 0.03111729584634304, "generator_grad_norm": 0.11556576192378998, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03931498527527, "peak_memory_bytes": 22207118336}
+{"step": 144, "generator_loss": 0.0247078575193882, "generator_grad_norm": 0.10638223588466644, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05579495429993, "peak_memory_bytes": 22207118336}
+{"step": 145, "generator_loss": 0.07313291728496552, "generator_grad_norm": 0.06300079822540283, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.95677900314331, "peak_memory_bytes": 22207118336}
+{"step": 146, "generator_loss": 0.01989143155515194, "generator_grad_norm": 0.1421123743057251, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08347177505493, "peak_memory_bytes": 22207118336}
+{"step": 147, "generator_loss": 0.05152301490306854, "generator_grad_norm": 0.06910934299230576, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.01727557182312, "peak_memory_bytes": 22207118336}
+{"step": 148, "generator_loss": 0.07280886173248291, "generator_grad_norm": 0.06529536843299866, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93700623512268, "peak_memory_bytes": 22207118336}
+{"step": 149, "generator_loss": 0.0342412069439888, "generator_grad_norm": 0.10006781667470932, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.06509304046631, "peak_memory_bytes": 22207118336}
+{"step": 150, "generator_loss": 0.06957927346229553, "generator_grad_norm": 0.08493683487176895, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.96635389328003, "peak_memory_bytes": 22207118336}
+{"step": 151, "generator_loss": 0.0340539775788784, "generator_grad_norm": 0.07114578038454056, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.0588903427124, "peak_memory_bytes": 22207118336}
+{"step": 152, "generator_loss": 0.14615726470947266, "generator_grad_norm": 0.07761552929878235, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.964560747146606, "peak_memory_bytes": 22207118336}
+{"step": 153, "generator_loss": 0.03208911046385765, "generator_grad_norm": 0.05410801246762276, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03104877471924, "peak_memory_bytes": 22207118336}
+{"step": 154, "generator_loss": 0.015478014945983887, "generator_grad_norm": 0.0655314177274704, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05896186828613, "peak_memory_bytes": 22207118336}
+{"step": 155, "generator_loss": 0.03527248650789261, "generator_grad_norm": 0.05972074717283249, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88253450393677, "peak_memory_bytes": 22207118336}
+{"step": 156, "generator_loss": 0.0344032384455204, "generator_grad_norm": 0.08164092898368835, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.02794790267944, "peak_memory_bytes": 22207118336}
+{"step": 157, "generator_loss": 0.034476615488529205, "generator_grad_norm": 0.06689576059579849, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.900821685791016, "peak_memory_bytes": 22207118336}
+{"step": 158, "generator_loss": 0.05317079648375511, "generator_grad_norm": 0.07933167368173599, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.04284167289734, "peak_memory_bytes": 22207118336}
+{"step": 159, "generator_loss": 0.02017601951956749, "generator_grad_norm": 0.055633388459682465, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93807935714722, "peak_memory_bytes": 22207118336}
+{"step": 160, "generator_loss": 0.024435291066765785, "generator_grad_norm": 0.06228254362940788, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.99989295005798, "peak_memory_bytes": 22207118336}
+{"step": 161, "generator_loss": 0.048426322638988495, "generator_grad_norm": 0.08381886035203934, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.112720251083374, "peak_memory_bytes": 22207118336}
+{"step": 162, "generator_loss": 0.06595819443464279, "generator_grad_norm": 0.05946244299411774, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.91275453567505, "peak_memory_bytes": 22207118336}
+{"step": 163, "generator_loss": 0.03688004985451698, "generator_grad_norm": 0.054807644337415695, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.14816927909851, "peak_memory_bytes": 22207118336}
+{"step": 164, "generator_loss": 0.022851306945085526, "generator_grad_norm": 0.06399118900299072, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.914671421051025, "peak_memory_bytes": 22207118336}
+{"step": 165, "generator_loss": 0.05659305676817894, "generator_grad_norm": 0.07911355048418045, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.010857820510864, "peak_memory_bytes": 22207118336}
+{"step": 166, "generator_loss": 0.028217226266860962, "generator_grad_norm": 0.09082085639238358, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.903247594833374, "peak_memory_bytes": 22207118336}
+{"step": 167, "generator_loss": 0.03798281028866768, "generator_grad_norm": 0.10842497646808624, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.07743835449219, "peak_memory_bytes": 22207118336}
+{"step": 168, "generator_loss": 0.015671726316213608, "generator_grad_norm": 0.09324725717306137, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.060718297958374, "peak_memory_bytes": 22207118336}
+{"step": 169, "generator_loss": 0.09173943847417831, "generator_grad_norm": 0.06418005377054214, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.89672350883484, "peak_memory_bytes": 22207118336}
+{"step": 170, "generator_loss": 0.0694725289940834, "generator_grad_norm": 0.06564024090766907, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.00446343421936, "peak_memory_bytes": 22207118336}
+{"step": 171, "generator_loss": 0.042225077748298645, "generator_grad_norm": 0.05359282344579697, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.902499198913574, "peak_memory_bytes": 22207118336}
+{"step": 172, "generator_loss": 0.023523541167378426, "generator_grad_norm": 0.06340853124856949, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.00808334350586, "peak_memory_bytes": 22207118336}
+{"step": 173, "generator_loss": 0.1186920702457428, "generator_grad_norm": 0.0793289840221405, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.94946026802063, "peak_memory_bytes": 22207118336}
+{"step": 174, "generator_loss": 0.03655270114541054, "generator_grad_norm": 0.0635564997792244, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.10817766189575, "peak_memory_bytes": 22207118336}
+{"step": 175, "generator_loss": 0.09101052582263947, "generator_grad_norm": 0.08019828796386719, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.997183322906494, "peak_memory_bytes": 22207118336}
+{"step": 176, "generator_loss": 0.04128626734018326, "generator_grad_norm": 0.06939037144184113, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.97967743873596, "peak_memory_bytes": 22207118336}
+{"step": 177, "generator_loss": 0.0782247930765152, "generator_grad_norm": 0.09201326966285706, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03376364707947, "peak_memory_bytes": 22207118336}
+{"step": 178, "generator_loss": 0.13857360184192657, "generator_grad_norm": 0.08956534415483475, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.888298749923706, "peak_memory_bytes": 22207118336}
+{"step": 179, "generator_loss": 0.04407915845513344, "generator_grad_norm": 0.056673984974622726, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03275012969971, "peak_memory_bytes": 22207118336}
+{"step": 180, "generator_loss": 0.0389353483915329, "generator_grad_norm": 0.07135889679193497, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03762125968933, "peak_memory_bytes": 22207118336}
+{"step": 181, "generator_loss": 0.028977172449231148, "generator_grad_norm": 0.06174766272306442, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8908257484436, "peak_memory_bytes": 22207118336}
+{"step": 182, "generator_loss": 0.10060428828001022, "generator_grad_norm": 0.078757144510746, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08073282241821, "peak_memory_bytes": 22207118336}
+{"step": 183, "generator_loss": 0.02858094498515129, "generator_grad_norm": 0.053234778344631195, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93537902832031, "peak_memory_bytes": 22207118336}
+{"step": 184, "generator_loss": 0.03874339535832405, "generator_grad_norm": 0.0739055648446083, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.018723011016846, "peak_memory_bytes": 22207118336}
+{"step": 185, "generator_loss": 0.05365682393312454, "generator_grad_norm": 0.076958067715168, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.93465185165405, "peak_memory_bytes": 22207118336}
+{"step": 186, "generator_loss": 0.044860731810331345, "generator_grad_norm": 0.06830932945013046, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.02738094329834, "peak_memory_bytes": 22207118336}
+{"step": 187, "generator_loss": 0.03903292119503021, "generator_grad_norm": 0.05690469965338707, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03613901138306, "peak_memory_bytes": 22207118336}
+{"step": 188, "generator_loss": 0.05954648181796074, "generator_grad_norm": 0.06584285199642181, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.90624785423279, "peak_memory_bytes": 22207118336}
+{"step": 189, "generator_loss": 0.03222750499844551, "generator_grad_norm": 0.07598879933357239, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03343749046326, "peak_memory_bytes": 22207118336}
+{"step": 190, "generator_loss": 0.07377743721008301, "generator_grad_norm": 0.09412325918674469, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.88999557495117, "peak_memory_bytes": 22207118336}
+{"step": 191, "generator_loss": 0.02874019369482994, "generator_grad_norm": 0.049843162298202515, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.08452820777893, "peak_memory_bytes": 22207118336}
+{"step": 192, "generator_loss": 0.022980550304055214, "generator_grad_norm": 0.05333597585558891, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.05485558509827, "peak_memory_bytes": 22207118336}
+{"step": 193, "generator_loss": 0.055883489549160004, "generator_grad_norm": 0.0871870368719101, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.91211700439453, "peak_memory_bytes": 22207118336}
+{"step": 194, "generator_loss": 0.08506827801465988, "generator_grad_norm": 0.08592723309993744, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03516983985901, "peak_memory_bytes": 22207118336}
+{"step": 195, "generator_loss": 0.05807612091302872, "generator_grad_norm": 0.0790700688958168, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.9613151550293, "peak_memory_bytes": 22207118336}
+{"step": 196, "generator_loss": 0.04578342288732529, "generator_grad_norm": 0.076502226293087, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03812885284424, "peak_memory_bytes": 22207118336}
+{"step": 197, "generator_loss": 0.07894004136323929, "generator_grad_norm": 0.0840674415230751, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.8878219127655, "peak_memory_bytes": 22207118336}
+{"step": 198, "generator_loss": 0.12087275087833405, "generator_grad_norm": 0.11643054336309433, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.04595446586609, "peak_memory_bytes": 22207118336}
+{"step": 199, "generator_loss": 0.02331393212080002, "generator_grad_norm": 0.0892941951751709, "temporal_rope_alpha": 1.0, "seconds_per_step": 34.03339886665344, "peak_memory_bytes": 22207118336}
+{"step": 200, "generator_loss": 0.03558746352791786, "generator_grad_norm": 0.07518526166677475, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.90160274505615, "peak_memory_bytes": 22207118336}

outputs/temporal_nope/a1_rope_cpt_lr2e7_probe200_seed20260805_kerrigan/validation_metrics.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "validation_loss": 0.06264185276813805, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 22.795755624771118, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 100, "validation_loss": 0.0514941630535759, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 23.613961696624756, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 200, "validation_loss": 0.04925253620604053, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 23.490389347076416, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}

outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt", "sha256": "c2c5fd1ddaf1431579e3f359e073f99e1b827ab450978dbf513434a5e52b7274", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt", "sha256": "c7728fb87b982489db2e718a8c7653882a2133dded77660b1cd804daf741a850", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/resolved_config.yaml

@@ -0,0 +1,123 @@
+experiment:
+  id: a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: RoPE-CPT matched control
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 1.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+  target_visible_lengths:
+  - 21
+  - 42
+  - 84
+  train_length: 21
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 4.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/train.log

@@ -0,0 +1,259 @@
+W0807 00:09:47.490000 3156067 site-packages/torch/distributed/run.py:793] 
+W0807 00:09:47.490000 3156067 site-packages/torch/distributed/run.py:793] *****************************************
+W0807 00:09:47.490000 3156067 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0807 00:09:47.490000 3156067 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank2]:[W807 00:09:57.868371707 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank4]:[W807 00:09:57.003604158 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank7]:[W807 00:09:57.611702039 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank1]:[W807 00:09:58.742944031 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W807 00:09:58.759281662 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W807 00:09:58.760208407 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank6]:[W807 00:09:58.821308742 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank0]:[W807 00:09:58.087043608 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.06264185 alpha=1.000000 samples=32 seconds=22.588
+step=1 loss=0.16898617 grad_norm=0.284440 alpha=1.000000 seconds=33.824
+step=2 loss=0.08452399 grad_norm=0.259132 alpha=1.000000 seconds=33.455
+step=3 loss=0.07878885 grad_norm=0.292787 alpha=1.000000 seconds=33.788
+step=4 loss=0.11278746 grad_norm=0.269995 alpha=1.000000 seconds=33.687
+step=5 loss=0.03989964 grad_norm=0.397074 alpha=1.000000 seconds=33.899
+step=6 loss=0.04870192 grad_norm=0.216642 alpha=1.000000 seconds=33.751
+step=7 loss=0.11759949 grad_norm=0.248604 alpha=1.000000 seconds=33.909
+step=8 loss=0.03943404 grad_norm=0.221221 alpha=1.000000 seconds=33.923
+step=9 loss=0.03800571 grad_norm=0.227170 alpha=1.000000 seconds=33.761
+step=10 loss=0.11269060 grad_norm=0.293723 alpha=1.000000 seconds=33.897
+step=11 loss=0.05024400 grad_norm=0.219113 alpha=1.000000 seconds=33.797
+step=12 loss=0.03831534 grad_norm=0.164259 alpha=1.000000 seconds=33.976
+step=13 loss=0.06634291 grad_norm=0.187611 alpha=1.000000 seconds=33.917
+step=14 loss=0.11188487 grad_norm=0.195986 alpha=1.000000 seconds=33.759
+step=15 loss=0.02364512 grad_norm=0.201556 alpha=1.000000 seconds=33.888
+step=16 loss=0.07214804 grad_norm=0.190197 alpha=1.000000 seconds=33.721
+step=17 loss=0.15304582 grad_norm=0.201078 alpha=1.000000 seconds=33.975
+step=18 loss=0.05604922 grad_norm=0.162735 alpha=1.000000 seconds=33.732
+step=19 loss=0.05080383 grad_norm=0.216206 alpha=1.000000 seconds=33.951
+step=20 loss=0.06704846 grad_norm=0.137741 alpha=1.000000 seconds=33.920
+step=21 loss=0.03732525 grad_norm=0.180131 alpha=1.000000 seconds=33.776
+step=22 loss=0.09984197 grad_norm=0.172156 alpha=1.000000 seconds=33.917
+step=23 loss=0.05177775 grad_norm=0.149101 alpha=1.000000 seconds=33.911
+step=24 loss=0.03230783 grad_norm=0.152521 alpha=1.000000 seconds=33.738
+step=25 loss=0.03134595 grad_norm=0.161357 alpha=1.000000 seconds=33.875
+step=26 loss=0.04738880 grad_norm=0.123898 alpha=1.000000 seconds=33.714
+step=27 loss=0.10072595 grad_norm=0.122465 alpha=1.000000 seconds=33.849
+step=28 loss=0.04306233 grad_norm=0.119941 alpha=1.000000 seconds=33.678
+step=29 loss=0.09208241 grad_norm=0.109408 alpha=1.000000 seconds=33.800
+step=30 loss=0.04000646 grad_norm=0.162170 alpha=1.000000 seconds=33.814
+step=31 loss=0.11359677 grad_norm=0.154410 alpha=1.000000 seconds=33.668
+step=32 loss=0.02177077 grad_norm=0.123483 alpha=1.000000 seconds=33.847
+step=33 loss=0.02864378 grad_norm=0.172702 alpha=1.000000 seconds=33.836
+step=34 loss=0.07808723 grad_norm=0.120447 alpha=1.000000 seconds=33.670
+step=35 loss=0.04009277 grad_norm=0.097723 alpha=1.000000 seconds=33.826
+step=36 loss=0.08138657 grad_norm=0.157169 alpha=1.000000 seconds=33.690
+step=37 loss=0.06744611 grad_norm=0.126092 alpha=1.000000 seconds=33.876
+step=38 loss=0.05399023 grad_norm=0.116229 alpha=1.000000 seconds=33.684
+step=39 loss=0.02866815 grad_norm=0.104800 alpha=1.000000 seconds=33.752
+step=40 loss=0.03080152 grad_norm=0.097288 alpha=1.000000 seconds=33.803
+step=41 loss=0.06157399 grad_norm=0.158684 alpha=1.000000 seconds=33.668
+step=42 loss=0.04387537 grad_norm=0.088382 alpha=1.000000 seconds=33.819
+step=43 loss=0.01868118 grad_norm=0.159113 alpha=1.000000 seconds=33.816
+step=44 loss=0.04954329 grad_norm=0.098418 alpha=1.000000 seconds=33.667
+step=45 loss=0.03947054 grad_norm=0.103766 alpha=1.000000 seconds=33.831
+step=46 loss=0.04017525 grad_norm=0.116644 alpha=1.000000 seconds=33.659
+step=47 loss=0.03709484 grad_norm=0.120793 alpha=1.000000 seconds=33.795
+step=48 loss=0.06521133 grad_norm=0.091717 alpha=1.000000 seconds=33.648
+step=49 loss=0.03468918 grad_norm=0.092241 alpha=1.000000 seconds=33.769
+step=50 loss=0.06914529 grad_norm=0.095890 alpha=1.000000 seconds=33.774
+step=51 loss=0.03005695 grad_norm=0.164944 alpha=1.000000 seconds=33.686
+step=52 loss=0.02717197 grad_norm=0.127388 alpha=1.000000 seconds=33.904
+step=53 loss=0.04516433 grad_norm=0.093121 alpha=1.000000 seconds=33.852
+step=54 loss=0.02781375 grad_norm=0.089038 alpha=1.000000 seconds=33.704
+step=55 loss=0.07292852 grad_norm=0.083980 alpha=1.000000 seconds=33.829
+step=56 loss=0.07090893 grad_norm=0.081273 alpha=1.000000 seconds=33.704
+step=57 loss=0.03508367 grad_norm=0.085408 alpha=1.000000 seconds=33.847
+step=58 loss=0.03228186 grad_norm=0.067989 alpha=1.000000 seconds=33.702
+step=59 loss=0.11147559 grad_norm=0.094198 alpha=1.000000 seconds=33.789
+step=60 loss=0.07833828 grad_norm=0.078146 alpha=1.000000 seconds=33.780
+step=61 loss=0.06684589 grad_norm=0.118361 alpha=1.000000 seconds=33.731
+step=62 loss=0.09972247 grad_norm=0.103581 alpha=1.000000 seconds=33.782
+step=63 loss=0.06998573 grad_norm=0.081561 alpha=1.000000 seconds=33.805
+step=64 loss=0.05400852 grad_norm=0.070429 alpha=1.000000 seconds=33.671
+step=65 loss=0.02168166 grad_norm=0.090696 alpha=1.000000 seconds=33.784
+step=66 loss=0.02514420 grad_norm=0.088192 alpha=1.000000 seconds=33.634
+step=67 loss=0.01399644 grad_norm=0.062871 alpha=1.000000 seconds=33.753
+step=68 loss=0.10250673 grad_norm=0.087630 alpha=1.000000 seconds=33.626
+step=69 loss=0.06609512 grad_norm=0.115754 alpha=1.000000 seconds=33.753
+step=70 loss=0.03636727 grad_norm=0.084714 alpha=1.000000 seconds=33.740
+step=71 loss=0.03416704 grad_norm=0.087931 alpha=1.000000 seconds=33.643
+step=72 loss=0.03955406 grad_norm=0.060765 alpha=1.000000 seconds=33.837
+step=73 loss=0.16107559 grad_norm=0.139679 alpha=1.000000 seconds=33.828
+step=74 loss=0.07246853 grad_norm=0.090469 alpha=1.000000 seconds=33.652
+step=75 loss=0.08827275 grad_norm=0.072928 alpha=1.000000 seconds=33.807
+step=76 loss=0.02481812 grad_norm=0.083885 alpha=1.000000 seconds=33.674
+step=77 loss=0.04250994 grad_norm=0.081548 alpha=1.000000 seconds=33.842
+step=78 loss=0.07153207 grad_norm=0.076064 alpha=1.000000 seconds=33.645
+step=79 loss=0.04667780 grad_norm=0.082680 alpha=1.000000 seconds=33.805
+step=80 loss=0.05478448 grad_norm=0.101728 alpha=1.000000 seconds=33.838
+step=81 loss=0.04710114 grad_norm=0.049445 alpha=1.000000 seconds=33.714
+step=82 loss=0.03440228 grad_norm=0.068088 alpha=1.000000 seconds=33.891
+step=83 loss=0.03663286 grad_norm=0.071631 alpha=1.000000 seconds=33.874
+step=84 loss=0.11668413 grad_norm=0.077906 alpha=1.000000 seconds=33.718
+step=85 loss=0.02754003 grad_norm=0.102958 alpha=1.000000 seconds=33.905
+step=86 loss=0.02144641 grad_norm=0.072169 alpha=1.000000 seconds=33.753
+step=87 loss=0.03314152 grad_norm=0.075109 alpha=1.000000 seconds=33.892
+step=88 loss=0.05843190 grad_norm=0.078377 alpha=1.000000 seconds=33.756
+step=89 loss=0.13036843 grad_norm=0.077829 alpha=1.000000 seconds=33.896
+step=90 loss=0.08905555 grad_norm=0.078515 alpha=1.000000 seconds=33.897
+step=91 loss=0.04199139 grad_norm=0.087249 alpha=1.000000 seconds=33.783
+step=92 loss=0.01975280 grad_norm=0.069604 alpha=1.000000 seconds=33.979
+step=93 loss=0.02038187 grad_norm=0.060615 alpha=1.000000 seconds=33.955
+step=94 loss=0.12310946 grad_norm=0.070583 alpha=1.000000 seconds=33.791
+step=95 loss=0.02475277 grad_norm=0.063172 alpha=1.000000 seconds=33.939
+step=96 loss=0.16603900 grad_norm=0.091364 alpha=1.000000 seconds=33.784
+step=97 loss=0.11731795 grad_norm=0.084316 alpha=1.000000 seconds=33.960
+step=98 loss=0.07396284 grad_norm=0.097570 alpha=1.000000 seconds=33.821
+step=99 loss=0.03720766 grad_norm=0.078341 alpha=1.000000 seconds=33.934
+step=100 loss=0.06482805 grad_norm=0.053541 alpha=1.000000 seconds=33.960
+validation step=100 loss=0.04924947 alpha=1.000000 samples=32 seconds=23.571
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+Start gathering distributed model states...Start gathering distributed model states...
+
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt sha256=c2c5fd1ddaf1431579e3f359e073f99e1b827ab450978dbf513434a5e52b7274
+step=101 loss=0.06047313 grad_norm=0.101640 alpha=1.000000 seconds=33.314
+step=102 loss=0.03314419 grad_norm=0.039925 alpha=1.000000 seconds=34.032
+step=103 loss=0.06584981 grad_norm=0.051526 alpha=1.000000 seconds=33.911
+step=104 loss=0.15614165 grad_norm=0.090788 alpha=1.000000 seconds=33.992
+step=105 loss=0.02304491 grad_norm=0.048750 alpha=1.000000 seconds=33.877
+step=106 loss=0.02174317 grad_norm=0.109381 alpha=1.000000 seconds=33.927
+step=107 loss=0.03242185 grad_norm=0.068463 alpha=1.000000 seconds=33.929
+step=108 loss=0.05999056 grad_norm=0.074218 alpha=1.000000 seconds=33.806
+step=109 loss=0.20279367 grad_norm=0.073653 alpha=1.000000 seconds=33.885
+step=110 loss=0.02755656 grad_norm=0.058382 alpha=1.000000 seconds=33.816
+step=111 loss=0.06812301 grad_norm=0.063358 alpha=1.000000 seconds=33.935
+step=112 loss=0.12751798 grad_norm=0.086588 alpha=1.000000 seconds=33.922
+step=113 loss=0.11873177 grad_norm=0.119058 alpha=1.000000 seconds=33.726
+step=114 loss=0.03264400 grad_norm=0.059145 alpha=1.000000 seconds=33.812
+step=115 loss=0.05770464 grad_norm=0.083270 alpha=1.000000 seconds=33.777
+step=116 loss=0.02914092 grad_norm=0.060887 alpha=1.000000 seconds=33.882
+step=117 loss=0.09453064 grad_norm=0.064526 alpha=1.000000 seconds=33.736
+step=118 loss=0.02094099 grad_norm=0.097403 alpha=1.000000 seconds=33.899
+step=119 loss=0.05481850 grad_norm=0.063041 alpha=1.000000 seconds=33.865
+step=120 loss=0.05235299 grad_norm=0.068656 alpha=1.000000 seconds=33.737
+step=121 loss=0.03724409 grad_norm=0.062908 alpha=1.000000 seconds=33.884
+step=122 loss=0.02299210 grad_norm=0.089691 alpha=1.000000 seconds=33.736
+step=123 loss=0.06395182 grad_norm=0.048005 alpha=1.000000 seconds=33.923
+step=124 loss=0.02509362 grad_norm=0.055295 alpha=1.000000 seconds=33.764
+step=125 loss=0.09612519 grad_norm=0.076681 alpha=1.000000 seconds=33.949
+step=126 loss=0.15624148 grad_norm=0.060934 alpha=1.000000 seconds=33.901
+step=127 loss=0.06359946 grad_norm=0.047415 alpha=1.000000 seconds=33.751
+step=128 loss=0.03499567 grad_norm=0.048506 alpha=1.000000 seconds=33.868
+step=129 loss=0.06494656 grad_norm=0.071666 alpha=1.000000 seconds=33.695
+step=130 loss=0.04831227 grad_norm=0.055034 alpha=1.000000 seconds=33.875
+step=131 loss=0.06111163 grad_norm=0.054629 alpha=1.000000 seconds=33.718
+step=132 loss=0.03500751 grad_norm=0.057663 alpha=1.000000 seconds=33.823
+step=133 loss=0.09345834 grad_norm=0.076068 alpha=1.000000 seconds=33.696
+step=134 loss=0.08717439 grad_norm=0.068667 alpha=1.000000 seconds=33.840
+step=135 loss=0.03031836 grad_norm=0.056907 alpha=1.000000 seconds=33.925
+step=136 loss=0.04779136 grad_norm=0.045026 alpha=1.000000 seconds=33.704
+step=137 loss=0.09216212 grad_norm=0.084991 alpha=1.000000 seconds=33.813
+step=138 loss=0.11667968 grad_norm=0.055501 alpha=1.000000 seconds=33.717
+step=139 loss=0.03757762 grad_norm=0.062936 alpha=1.000000 seconds=33.866
+step=140 loss=0.01502442 grad_norm=0.075720 alpha=1.000000 seconds=33.838
+step=141 loss=0.04696148 grad_norm=0.061885 alpha=1.000000 seconds=33.716
+step=142 loss=0.09654515 grad_norm=0.055182 alpha=1.000000 seconds=33.790
+step=143 loss=0.03018590 grad_norm=0.090426 alpha=1.000000 seconds=33.756
+step=144 loss=0.02311911 grad_norm=0.089579 alpha=1.000000 seconds=33.876
+step=145 loss=0.07085357 grad_norm=0.041061 alpha=1.000000 seconds=33.733
+step=146 loss=0.01922528 grad_norm=0.119706 alpha=1.000000 seconds=33.876
+step=147 loss=0.04984973 grad_norm=0.047553 alpha=1.000000 seconds=33.808
+step=148 loss=0.07071530 grad_norm=0.043429 alpha=1.000000 seconds=33.759
+step=149 loss=0.03286100 grad_norm=0.071598 alpha=1.000000 seconds=33.839
+step=150 loss=0.06808022 grad_norm=0.063889 alpha=1.000000 seconds=33.705
+step=151 loss=0.03252810 grad_norm=0.046533 alpha=1.000000 seconds=33.856
+step=152 loss=0.14236210 grad_norm=0.059039 alpha=1.000000 seconds=33.757
+step=153 loss=0.03068572 grad_norm=0.038693 alpha=1.000000 seconds=33.836
+step=154 loss=0.01450500 grad_norm=0.050424 alpha=1.000000 seconds=33.817
+step=155 loss=0.03293972 grad_norm=0.047150 alpha=1.000000 seconds=33.659
+step=156 loss=0.03323923 grad_norm=0.063373 alpha=1.000000 seconds=33.809
+step=157 loss=0.03337437 grad_norm=0.054688 alpha=1.000000 seconds=33.695
+step=158 loss=0.05103671 grad_norm=0.062788 alpha=1.000000 seconds=33.824
+step=159 loss=0.01945036 grad_norm=0.046738 alpha=1.000000 seconds=33.696
+step=160 loss=0.02356442 grad_norm=0.046201 alpha=1.000000 seconds=33.788
+step=161 loss=0.04674975 grad_norm=0.062285 alpha=1.000000 seconds=33.867
+step=162 loss=0.06389064 grad_norm=0.041322 alpha=1.000000 seconds=33.689
+step=163 loss=0.03569222 grad_norm=0.051518 alpha=1.000000 seconds=33.905
+step=164 loss=0.02200918 grad_norm=0.044516 alpha=1.000000 seconds=33.674
+step=165 loss=0.05415873 grad_norm=0.053278 alpha=1.000000 seconds=33.720
+step=166 loss=0.02712130 grad_norm=0.073821 alpha=1.000000 seconds=33.653
+step=167 loss=0.03663151 grad_norm=0.079821 alpha=1.000000 seconds=33.788
+step=168 loss=0.01470700 grad_norm=0.068922 alpha=1.000000 seconds=33.793
+step=169 loss=0.08892927 grad_norm=0.049588 alpha=1.000000 seconds=33.641
+step=170 loss=0.06714903 grad_norm=0.049045 alpha=1.000000 seconds=33.761
+step=171 loss=0.04063438 grad_norm=0.036584 alpha=1.000000 seconds=33.682
+step=172 loss=0.02290949 grad_norm=0.046203 alpha=1.000000 seconds=33.792
+step=173 loss=0.11500911 grad_norm=0.062547 alpha=1.000000 seconds=33.681
+step=174 loss=0.03536972 grad_norm=0.046062 alpha=1.000000 seconds=33.775
+step=175 loss=0.08778166 grad_norm=0.070992 alpha=1.000000 seconds=33.752
+step=176 loss=0.04016441 grad_norm=0.084677 alpha=1.000000 seconds=33.674
+step=177 loss=0.07587081 grad_norm=0.078414 alpha=1.000000 seconds=33.806
+step=178 loss=0.13488907 grad_norm=0.194692 alpha=1.000000 seconds=33.644
+step=179 loss=0.04282106 grad_norm=0.042093 alpha=1.000000 seconds=33.860
+step=180 loss=0.03752256 grad_norm=0.061931 alpha=1.000000 seconds=33.763
+step=181 loss=0.02747862 grad_norm=0.040434 alpha=1.000000 seconds=33.677
+step=182 loss=0.09777991 grad_norm=0.058422 alpha=1.000000 seconds=33.803
+step=183 loss=0.02759719 grad_norm=0.035335 alpha=1.000000 seconds=33.693
+step=184 loss=0.03785538 grad_norm=0.061185 alpha=1.000000 seconds=33.797
+step=185 loss=0.05184986 grad_norm=0.052261 alpha=1.000000 seconds=33.713
+step=186 loss=0.04354292 grad_norm=0.054049 alpha=1.000000 seconds=33.873
+step=187 loss=0.03777561 grad_norm=0.046477 alpha=1.000000 seconds=33.800
+step=188 loss=0.05764690 grad_norm=0.043085 alpha=1.000000 seconds=33.666
+step=189 loss=0.03093739 grad_norm=0.074260 alpha=1.000000 seconds=33.775
+step=190 loss=0.07192354 grad_norm=0.080684 alpha=1.000000 seconds=33.632
+step=191 loss=0.02779631 grad_norm=0.034895 alpha=1.000000 seconds=33.793
+step=192 loss=0.02213561 grad_norm=0.038625 alpha=1.000000 seconds=33.811
+step=193 loss=0.05410427 grad_norm=0.099049 alpha=1.000000 seconds=33.653
+step=194 loss=0.08320453 grad_norm=0.080664 alpha=1.000000 seconds=33.801
+step=195 loss=0.05623063 grad_norm=0.066331 alpha=1.000000 seconds=33.647
+step=196 loss=0.04387650 grad_norm=0.067425 alpha=1.000000 seconds=33.828
+step=197 loss=0.07663434 grad_norm=0.071194 alpha=1.000000 seconds=33.694
+step=198 loss=0.11742631 grad_norm=0.129569 alpha=1.000000 seconds=33.851
+step=199 loss=0.02272441 grad_norm=0.078225 alpha=1.000000 seconds=33.942
+step=200 loss=0.03442148 grad_norm=0.061388 alpha=1.000000 seconds=33.682
+validation step=200 loss=0.04765816 alpha=1.000000 samples=32 seconds=23.295
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a1_rope_cpt_lr4e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt sha256=c7728fb87b982489db2e718a8c7653882a2133dded77660b1cd804daf741a850

outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt", "sha256": "4213a600500a2da1b8698e8cd81d7667925097753abea35f8f799aca605d0d99", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt", "sha256": "cdca71bccd2f76b94120fc51ede62931841283d3a5fd414ad87f5095c0a7ff74", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/resolved_config.yaml

@@ -0,0 +1,123 @@
+experiment:
+  id: a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: Hard Temporal DroPE
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: none
+  temporal_rope_alpha: 0.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+  target_visible_lengths:
+  - 21
+  - 42
+  - 84
+  train_length: 21
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 1.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/train.log

@@ -0,0 +1,259 @@
+W0806 13:33:39.419000 2847191 site-packages/torch/distributed/run.py:793] 
+W0806 13:33:39.419000 2847191 site-packages/torch/distributed/run.py:793] *****************************************
+W0806 13:33:39.419000 2847191 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0806 13:33:39.419000 2847191 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank1]:[W806 13:33:48.318119518 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank4]:[W806 13:33:49.324686130 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank7]:[W806 13:33:49.341972336 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank6]:[W806 13:33:49.364484262 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank2]:[W806 13:33:49.389167570 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W806 13:33:49.438788336 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W806 13:33:49.464334781 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank0]:[W806 13:33:49.626436129 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.26354822 alpha=0.000000 samples=32 seconds=22.944
+step=1 loss=0.75935781 grad_norm=2.223690 alpha=0.000000 seconds=33.455
+step=2 loss=0.60527384 grad_norm=2.966155 alpha=0.000000 seconds=33.660
+step=3 loss=0.35504434 grad_norm=2.872042 alpha=0.000000 seconds=34.018
+step=4 loss=0.44725409 grad_norm=2.800238 alpha=0.000000 seconds=33.949
+step=5 loss=0.13983518 grad_norm=2.238496 alpha=0.000000 seconds=34.141
+step=6 loss=0.15167883 grad_norm=2.219674 alpha=0.000000 seconds=34.030
+step=7 loss=0.56869608 grad_norm=2.971950 alpha=0.000000 seconds=34.180
+step=8 loss=0.32047978 grad_norm=2.339395 alpha=0.000000 seconds=34.093
+step=9 loss=0.20799437 grad_norm=2.727454 alpha=0.000000 seconds=33.967
+step=10 loss=0.73815399 grad_norm=2.712714 alpha=0.000000 seconds=34.141
+step=11 loss=0.20480685 grad_norm=2.472412 alpha=0.000000 seconds=34.046
+step=12 loss=0.24539575 grad_norm=2.069215 alpha=0.000000 seconds=34.187
+step=13 loss=0.54612887 grad_norm=2.675020 alpha=0.000000 seconds=34.224
+step=14 loss=0.60367477 grad_norm=2.595930 alpha=0.000000 seconds=34.058
+step=15 loss=0.08414430 grad_norm=2.755935 alpha=0.000000 seconds=34.190
+step=16 loss=0.15145978 grad_norm=2.055475 alpha=0.000000 seconds=34.079
+step=17 loss=0.44897553 grad_norm=2.598983 alpha=0.000000 seconds=34.209
+step=18 loss=0.30164206 grad_norm=2.395159 alpha=0.000000 seconds=33.990
+step=19 loss=0.19252886 grad_norm=2.485991 alpha=0.000000 seconds=34.130
+step=20 loss=0.42920291 grad_norm=2.027293 alpha=0.000000 seconds=34.133
+step=21 loss=0.14726152 grad_norm=2.568621 alpha=0.000000 seconds=34.050
+step=22 loss=0.49208450 grad_norm=2.562724 alpha=0.000000 seconds=34.182
+step=23 loss=0.35671222 grad_norm=2.550663 alpha=0.000000 seconds=34.196
+step=24 loss=0.29054472 grad_norm=2.179967 alpha=0.000000 seconds=34.065
+step=25 loss=0.09976621 grad_norm=1.991174 alpha=0.000000 seconds=34.169
+step=26 loss=0.24499948 grad_norm=2.025902 alpha=0.000000 seconds=34.062
+step=27 loss=0.31936511 grad_norm=2.089768 alpha=0.000000 seconds=34.204
+step=28 loss=0.10136045 grad_norm=2.064397 alpha=0.000000 seconds=33.972
+step=29 loss=0.45983842 grad_norm=1.926861 alpha=0.000000 seconds=34.113
+step=30 loss=0.19620059 grad_norm=2.378699 alpha=0.000000 seconds=34.080
+step=31 loss=0.58350462 grad_norm=2.161548 alpha=0.000000 seconds=34.047
+step=32 loss=0.11053171 grad_norm=2.448980 alpha=0.000000 seconds=34.243
+step=33 loss=0.20926458 grad_norm=2.077834 alpha=0.000000 seconds=34.219
+step=34 loss=0.47374880 grad_norm=2.044412 alpha=0.000000 seconds=34.038
+step=35 loss=0.15037584 grad_norm=1.714919 alpha=0.000000 seconds=34.187
+step=36 loss=0.29027894 grad_norm=2.060817 alpha=0.000000 seconds=34.032
+step=37 loss=0.43730047 grad_norm=2.468075 alpha=0.000000 seconds=34.275
+step=38 loss=0.20082363 grad_norm=1.859315 alpha=0.000000 seconds=34.002
+step=39 loss=0.17684376 grad_norm=2.144067 alpha=0.000000 seconds=34.058
+step=40 loss=0.19853067 grad_norm=1.936963 alpha=0.000000 seconds=34.125
+step=41 loss=0.26339567 grad_norm=2.188529 alpha=0.000000 seconds=33.989
+step=42 loss=0.21096048 grad_norm=1.899626 alpha=0.000000 seconds=34.110
+step=43 loss=0.09435697 grad_norm=2.380459 alpha=0.000000 seconds=34.149
+step=44 loss=0.27617213 grad_norm=2.091246 alpha=0.000000 seconds=34.029
+step=45 loss=0.18252823 grad_norm=1.901483 alpha=0.000000 seconds=34.184
+step=46 loss=0.13081977 grad_norm=2.135986 alpha=0.000000 seconds=34.058
+step=47 loss=0.17839144 grad_norm=2.172694 alpha=0.000000 seconds=34.270
+step=48 loss=0.21894024 grad_norm=1.567966 alpha=0.000000 seconds=34.020
+step=49 loss=0.22098237 grad_norm=1.683347 alpha=0.000000 seconds=34.124
+step=50 loss=0.23106001 grad_norm=2.092800 alpha=0.000000 seconds=34.095
+step=51 loss=0.13642560 grad_norm=1.749455 alpha=0.000000 seconds=33.986
+step=52 loss=0.19232206 grad_norm=2.308356 alpha=0.000000 seconds=34.115
+step=53 loss=0.22942403 grad_norm=1.950499 alpha=0.000000 seconds=34.101
+step=54 loss=0.10468858 grad_norm=2.345900 alpha=0.000000 seconds=33.963
+step=55 loss=0.20278865 grad_norm=1.765538 alpha=0.000000 seconds=34.124
+step=56 loss=0.34531826 grad_norm=2.082208 alpha=0.000000 seconds=33.999
+step=57 loss=0.10997936 grad_norm=1.897818 alpha=0.000000 seconds=34.177
+step=58 loss=0.15457369 grad_norm=1.794075 alpha=0.000000 seconds=34.042
+step=59 loss=0.20693515 grad_norm=1.659382 alpha=0.000000 seconds=34.114
+step=60 loss=0.46453458 grad_norm=2.120290 alpha=0.000000 seconds=34.202
+step=61 loss=0.16497031 grad_norm=2.161371 alpha=0.000000 seconds=33.956
+step=62 loss=0.50519973 grad_norm=2.095455 alpha=0.000000 seconds=34.141
+step=63 loss=0.27113634 grad_norm=1.678140 alpha=0.000000 seconds=34.127
+step=64 loss=0.21510738 grad_norm=2.016161 alpha=0.000000 seconds=34.024
+step=65 loss=0.07921797 grad_norm=1.723998 alpha=0.000000 seconds=34.150
+step=66 loss=0.11020257 grad_norm=1.553360 alpha=0.000000 seconds=33.992
+step=67 loss=0.10587035 grad_norm=2.006851 alpha=0.000000 seconds=34.116
+step=68 loss=0.45250821 grad_norm=1.892137 alpha=0.000000 seconds=33.964
+step=69 loss=0.27824828 grad_norm=2.373800 alpha=0.000000 seconds=34.094
+step=70 loss=0.15435009 grad_norm=1.782045 alpha=0.000000 seconds=34.054
+step=71 loss=0.16790250 grad_norm=2.016589 alpha=0.000000 seconds=34.044
+step=72 loss=0.18019678 grad_norm=1.443899 alpha=0.000000 seconds=34.149
+step=73 loss=0.56936902 grad_norm=1.953657 alpha=0.000000 seconds=34.079
+step=74 loss=0.40548044 grad_norm=1.687200 alpha=0.000000 seconds=33.900
+step=75 loss=0.39971569 grad_norm=1.816321 alpha=0.000000 seconds=34.070
+step=76 loss=0.14739259 grad_norm=1.621850 alpha=0.000000 seconds=33.958
+step=77 loss=0.26764172 grad_norm=1.991485 alpha=0.000000 seconds=34.123
+step=78 loss=0.36442953 grad_norm=1.904644 alpha=0.000000 seconds=33.926
+step=79 loss=0.21102826 grad_norm=1.909206 alpha=0.000000 seconds=34.117
+step=80 loss=0.31315073 grad_norm=1.835116 alpha=0.000000 seconds=34.053
+step=81 loss=0.12715329 grad_norm=1.102603 alpha=0.000000 seconds=34.049
+step=82 loss=0.28607231 grad_norm=1.921154 alpha=0.000000 seconds=34.091
+step=83 loss=0.12720156 grad_norm=1.515612 alpha=0.000000 seconds=34.114
+step=84 loss=0.60566223 grad_norm=1.811714 alpha=0.000000 seconds=34.045
+step=85 loss=0.12675895 grad_norm=2.116135 alpha=0.000000 seconds=34.153
+step=86 loss=0.05427943 grad_norm=1.692894 alpha=0.000000 seconds=33.999
+step=87 loss=0.20486155 grad_norm=1.980087 alpha=0.000000 seconds=34.158
+step=88 loss=0.12747498 grad_norm=1.735975 alpha=0.000000 seconds=33.961
+step=89 loss=0.51247042 grad_norm=1.621820 alpha=0.000000 seconds=34.188
+step=90 loss=0.34770143 grad_norm=1.513935 alpha=0.000000 seconds=34.187
+step=91 loss=0.14166974 grad_norm=1.477237 alpha=0.000000 seconds=33.982
+step=92 loss=0.09116819 grad_norm=2.111791 alpha=0.000000 seconds=34.117
+step=93 loss=0.09987239 grad_norm=1.069182 alpha=0.000000 seconds=34.131
+step=94 loss=0.47033310 grad_norm=1.622569 alpha=0.000000 seconds=33.988
+step=95 loss=0.06501450 grad_norm=1.577457 alpha=0.000000 seconds=34.098
+step=96 loss=0.51227808 grad_norm=1.711941 alpha=0.000000 seconds=34.064
+step=97 loss=0.44193295 grad_norm=1.372194 alpha=0.000000 seconds=34.153
+step=98 loss=0.21266872 grad_norm=2.357080 alpha=0.000000 seconds=34.002
+step=99 loss=0.12523443 grad_norm=2.306888 alpha=0.000000 seconds=34.163
+step=100 loss=0.19680248 grad_norm=1.714960 alpha=0.000000 seconds=34.125
+validation step=100 loss=0.19042654 alpha=0.000000 samples=32 seconds=23.483
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states.../home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+
+Start gathering distributed model states.../home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt sha256=4213a600500a2da1b8698e8cd81d7667925097753abea35f8f799aca605d0d99
+step=101 loss=0.27869985 grad_norm=2.395654 alpha=0.000000 seconds=33.510
+step=102 loss=0.16974846 grad_norm=1.247127 alpha=0.000000 seconds=34.118
+step=103 loss=0.11542259 grad_norm=1.250616 alpha=0.000000 seconds=33.996
+step=104 loss=0.54958230 grad_norm=2.166646 alpha=0.000000 seconds=34.158
+step=105 loss=0.07991378 grad_norm=1.552543 alpha=0.000000 seconds=33.985
+step=106 loss=0.07704064 grad_norm=2.321823 alpha=0.000000 seconds=34.125
+step=107 loss=0.09948509 grad_norm=1.275812 alpha=0.000000 seconds=34.097
+step=108 loss=0.25519207 grad_norm=1.985957 alpha=0.000000 seconds=33.970
+step=109 loss=0.66361499 grad_norm=2.106556 alpha=0.000000 seconds=34.035
+step=110 loss=0.09088703 grad_norm=1.835997 alpha=0.000000 seconds=33.982
+step=111 loss=0.31744298 grad_norm=1.528121 alpha=0.000000 seconds=34.123
+step=112 loss=0.36545137 grad_norm=2.223571 alpha=0.000000 seconds=34.174
+step=113 loss=0.33994472 grad_norm=1.672112 alpha=0.000000 seconds=33.956
+step=114 loss=0.18649156 grad_norm=1.830313 alpha=0.000000 seconds=34.203
+step=115 loss=0.12383473 grad_norm=1.620675 alpha=0.000000 seconds=33.969
+step=116 loss=0.09826242 grad_norm=1.228240 alpha=0.000000 seconds=34.103
+step=117 loss=0.39745894 grad_norm=1.588117 alpha=0.000000 seconds=33.906
+step=118 loss=0.06591198 grad_norm=1.323338 alpha=0.000000 seconds=34.062
+step=119 loss=0.16710764 grad_norm=1.050651 alpha=0.000000 seconds=34.101
+step=120 loss=0.32538214 grad_norm=1.532550 alpha=0.000000 seconds=33.996
+step=121 loss=0.17096138 grad_norm=1.656404 alpha=0.000000 seconds=34.208
+step=122 loss=0.09711315 grad_norm=1.493304 alpha=0.000000 seconds=33.966
+step=123 loss=0.21705703 grad_norm=1.940650 alpha=0.000000 seconds=34.170
+step=124 loss=0.11210376 grad_norm=1.531108 alpha=0.000000 seconds=33.975
+step=125 loss=0.28281236 grad_norm=1.719060 alpha=0.000000 seconds=34.138
+step=126 loss=0.44289857 grad_norm=1.367165 alpha=0.000000 seconds=34.110
+step=127 loss=0.20413579 grad_norm=0.997287 alpha=0.000000 seconds=33.941
+step=128 loss=0.13794345 grad_norm=1.372594 alpha=0.000000 seconds=34.080
+step=129 loss=0.31890410 grad_norm=1.855719 alpha=0.000000 seconds=33.934
+step=130 loss=0.14168894 grad_norm=1.579080 alpha=0.000000 seconds=34.135
+step=131 loss=0.36776012 grad_norm=1.745346 alpha=0.000000 seconds=34.029
+step=132 loss=0.19775671 grad_norm=1.389308 alpha=0.000000 seconds=34.152
+step=133 loss=0.36153823 grad_norm=2.027764 alpha=0.000000 seconds=34.108
+step=134 loss=0.28720716 grad_norm=1.955605 alpha=0.000000 seconds=34.111
+step=135 loss=0.07775564 grad_norm=1.253937 alpha=0.000000 seconds=34.108
+step=136 loss=0.19153304 grad_norm=1.687629 alpha=0.000000 seconds=33.945
+step=137 loss=0.28437427 grad_norm=2.054076 alpha=0.000000 seconds=34.053
+step=138 loss=0.29153678 grad_norm=1.267379 alpha=0.000000 seconds=33.932
+step=139 loss=0.11643425 grad_norm=1.295083 alpha=0.000000 seconds=34.116
+step=140 loss=0.07979386 grad_norm=1.419625 alpha=0.000000 seconds=34.095
+step=141 loss=0.19638139 grad_norm=2.178253 alpha=0.000000 seconds=33.972
+step=142 loss=0.49552900 grad_norm=1.229861 alpha=0.000000 seconds=34.037
+step=143 loss=0.06325501 grad_norm=1.750996 alpha=0.000000 seconds=34.030
+step=144 loss=0.05860052 grad_norm=2.357242 alpha=0.000000 seconds=34.127
+step=145 loss=0.16697270 grad_norm=1.451675 alpha=0.000000 seconds=33.970
+step=146 loss=0.04920511 grad_norm=1.549153 alpha=0.000000 seconds=34.154
+step=147 loss=0.27673495 grad_norm=1.213994 alpha=0.000000 seconds=34.073
+step=148 loss=0.12903348 grad_norm=1.527614 alpha=0.000000 seconds=34.008
+step=149 loss=0.11807132 grad_norm=1.705578 alpha=0.000000 seconds=34.116
+step=150 loss=0.18645105 grad_norm=1.471327 alpha=0.000000 seconds=33.995
+step=151 loss=0.08747143 grad_norm=1.717180 alpha=0.000000 seconds=34.109
+step=152 loss=0.46293309 grad_norm=1.529763 alpha=0.000000 seconds=33.971
+step=153 loss=0.10276111 grad_norm=1.076084 alpha=0.000000 seconds=34.083
+step=154 loss=0.05859535 grad_norm=1.363401 alpha=0.000000 seconds=34.143
+step=155 loss=0.10235023 grad_norm=2.233249 alpha=0.000000 seconds=33.970
+step=156 loss=0.12837346 grad_norm=1.916502 alpha=0.000000 seconds=34.183
+step=157 loss=0.13724846 grad_norm=1.493536 alpha=0.000000 seconds=34.096
+step=158 loss=0.10979569 grad_norm=1.339102 alpha=0.000000 seconds=34.208
+step=159 loss=0.11822557 grad_norm=1.334910 alpha=0.000000 seconds=34.001
+step=160 loss=0.06709519 grad_norm=1.258438 alpha=0.000000 seconds=34.075
+step=161 loss=0.12811759 grad_norm=1.430369 alpha=0.000000 seconds=34.172
+step=162 loss=0.18193582 grad_norm=1.112770 alpha=0.000000 seconds=34.044
+step=163 loss=0.11109939 grad_norm=0.956356 alpha=0.000000 seconds=34.122
+step=164 loss=0.10705547 grad_norm=1.062125 alpha=0.000000 seconds=34.007
+step=165 loss=0.09903625 grad_norm=1.433396 alpha=0.000000 seconds=33.993
+step=166 loss=0.08583971 grad_norm=1.548965 alpha=0.000000 seconds=34.041
+step=167 loss=0.19312160 grad_norm=1.221882 alpha=0.000000 seconds=34.094
+step=168 loss=0.03943687 grad_norm=1.470906 alpha=0.000000 seconds=34.161
+step=169 loss=0.26142320 grad_norm=1.462252 alpha=0.000000 seconds=34.079
+step=170 loss=0.13403128 grad_norm=1.199995 alpha=0.000000 seconds=34.241
+step=171 loss=0.20547889 grad_norm=1.227654 alpha=0.000000 seconds=34.062
+step=172 loss=0.07037388 grad_norm=1.428296 alpha=0.000000 seconds=34.158
+step=173 loss=0.35901532 grad_norm=1.276779 alpha=0.000000 seconds=33.986
+step=174 loss=0.11954743 grad_norm=0.989415 alpha=0.000000 seconds=34.109
+step=175 loss=0.21536748 grad_norm=1.348248 alpha=0.000000 seconds=34.067
+step=176 loss=0.11523949 grad_norm=1.700210 alpha=0.000000 seconds=33.963
+step=177 loss=0.25504780 grad_norm=1.310045 alpha=0.000000 seconds=34.110
+step=178 loss=0.55237728 grad_norm=1.694701 alpha=0.000000 seconds=34.046
+step=179 loss=0.09759549 grad_norm=1.011915 alpha=0.000000 seconds=34.194
+step=180 loss=0.11423352 grad_norm=1.476003 alpha=0.000000 seconds=34.136
+step=181 loss=0.07288439 grad_norm=1.381628 alpha=0.000000 seconds=34.043
+step=182 loss=0.37147442 grad_norm=2.207666 alpha=0.000000 seconds=34.170
+step=183 loss=0.08936927 grad_norm=0.822976 alpha=0.000000 seconds=34.089
+step=184 loss=0.15965757 grad_norm=1.809371 alpha=0.000000 seconds=34.141
+step=185 loss=0.18111168 grad_norm=1.807492 alpha=0.000000 seconds=34.020
+step=186 loss=0.16450234 grad_norm=1.281193 alpha=0.000000 seconds=34.221
+step=187 loss=0.15172023 grad_norm=1.252789 alpha=0.000000 seconds=34.126
+step=188 loss=0.19916776 grad_norm=1.343523 alpha=0.000000 seconds=34.005
+step=189 loss=0.16932426 grad_norm=1.381966 alpha=0.000000 seconds=34.103
+step=190 loss=0.23398858 grad_norm=1.663133 alpha=0.000000 seconds=34.081
+step=191 loss=0.06716238 grad_norm=0.954076 alpha=0.000000 seconds=34.149
+step=192 loss=0.05351517 grad_norm=0.898977 alpha=0.000000 seconds=34.265
+step=193 loss=0.28825223 grad_norm=1.822443 alpha=0.000000 seconds=34.075
+step=194 loss=0.25262895 grad_norm=1.737438 alpha=0.000000 seconds=34.180
+step=195 loss=0.10312087 grad_norm=1.296191 alpha=0.000000 seconds=34.043
+step=196 loss=0.11589099 grad_norm=0.978812 alpha=0.000000 seconds=34.212
+step=197 loss=0.27396035 grad_norm=1.364575 alpha=0.000000 seconds=33.997
+step=198 loss=0.39362696 grad_norm=1.717740 alpha=0.000000 seconds=34.142
+step=199 loss=0.07802467 grad_norm=1.635959 alpha=0.000000 seconds=34.127
+step=200 loss=0.07862440 grad_norm=1.314411 alpha=0.000000 seconds=33.986
+validation step=200 loss=0.14632052 alpha=0.000000 samples=32 seconds=23.644
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...Start gathering distributed model states...
+
+Start gathering distributed model states...
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt sha256=cdca71bccd2f76b94120fc51ede62931841283d3a5fd414ad87f5095c0a7ff74

outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/training_metrics.jsonl

@@ -0,0 +1,200 @@
+{"step": 1, "generator_loss": 0.7593578100204468, "generator_grad_norm": 2.2236902713775635, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.45545434951782, "peak_memory_bytes": 20785906176}
+{"step": 2, "generator_loss": 0.6052738428115845, "generator_grad_norm": 2.9661550521850586, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66044545173645, "peak_memory_bytes": 22206258176}
+{"step": 3, "generator_loss": 0.35504433512687683, "generator_grad_norm": 2.872042179107666, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.017749071121216, "peak_memory_bytes": 22206258176}
+{"step": 4, "generator_loss": 0.44725409150123596, "generator_grad_norm": 2.8002383708953857, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94925117492676, "peak_memory_bytes": 22206258176}
+{"step": 5, "generator_loss": 0.1398351788520813, "generator_grad_norm": 2.2384955883026123, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14119052886963, "peak_memory_bytes": 22206694400}
+{"step": 6, "generator_loss": 0.15167883038520813, "generator_grad_norm": 2.2196741104125977, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.029566049575806, "peak_memory_bytes": 22206694400}
+{"step": 7, "generator_loss": 0.5686960816383362, "generator_grad_norm": 2.9719502925872803, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17985963821411, "peak_memory_bytes": 22206694400}
+{"step": 8, "generator_loss": 0.32047978043556213, "generator_grad_norm": 2.3393945693969727, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09349608421326, "peak_memory_bytes": 22206694400}
+{"step": 9, "generator_loss": 0.20799437165260315, "generator_grad_norm": 2.72745418548584, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96666121482849, "peak_memory_bytes": 22206694400}
+{"step": 10, "generator_loss": 0.7381539940834045, "generator_grad_norm": 2.712714433670044, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14075827598572, "peak_memory_bytes": 22206694400}
+{"step": 11, "generator_loss": 0.204806849360466, "generator_grad_norm": 2.472411870956421, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04580307006836, "peak_memory_bytes": 22206694400}
+{"step": 12, "generator_loss": 0.2453957498073578, "generator_grad_norm": 2.0692152976989746, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1870539188385, "peak_memory_bytes": 22206694400}
+{"step": 13, "generator_loss": 0.5461288690567017, "generator_grad_norm": 2.6750199794769287, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.224199295043945, "peak_memory_bytes": 22206694400}
+{"step": 14, "generator_loss": 0.6036747694015503, "generator_grad_norm": 2.5959296226501465, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.058085918426514, "peak_memory_bytes": 22206694400}
+{"step": 15, "generator_loss": 0.08414430171251297, "generator_grad_norm": 2.7559351921081543, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.19021797180176, "peak_memory_bytes": 22206694400}
+{"step": 16, "generator_loss": 0.15145978331565857, "generator_grad_norm": 2.0554745197296143, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0791289806366, "peak_memory_bytes": 22206694400}
+{"step": 17, "generator_loss": 0.448975533246994, "generator_grad_norm": 2.598982810974121, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20867156982422, "peak_memory_bytes": 22206694400}
+{"step": 18, "generator_loss": 0.3016420602798462, "generator_grad_norm": 2.3951592445373535, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99014401435852, "peak_memory_bytes": 22206694400}
+{"step": 19, "generator_loss": 0.1925288587808609, "generator_grad_norm": 2.4859912395477295, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.129584074020386, "peak_memory_bytes": 22206694400}
+{"step": 20, "generator_loss": 0.4292029142379761, "generator_grad_norm": 2.0272932052612305, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.13314890861511, "peak_memory_bytes": 22206831616}
+{"step": 21, "generator_loss": 0.14726151525974274, "generator_grad_norm": 2.5686209201812744, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04954195022583, "peak_memory_bytes": 22206831616}
+{"step": 22, "generator_loss": 0.4920845031738281, "generator_grad_norm": 2.5627241134643555, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18175792694092, "peak_memory_bytes": 22206831616}
+{"step": 23, "generator_loss": 0.3567122220993042, "generator_grad_norm": 2.5506632328033447, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.19642972946167, "peak_memory_bytes": 22206981120}
+{"step": 24, "generator_loss": 0.290544718503952, "generator_grad_norm": 2.179967164993286, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.064541816711426, "peak_memory_bytes": 22206981120}
+{"step": 25, "generator_loss": 0.09976620972156525, "generator_grad_norm": 1.9911744594573975, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16862344741821, "peak_memory_bytes": 22206981120}
+{"step": 26, "generator_loss": 0.2449994832277298, "generator_grad_norm": 2.025902271270752, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0624897480011, "peak_memory_bytes": 22206981120}
+{"step": 27, "generator_loss": 0.31936511397361755, "generator_grad_norm": 2.089768409729004, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20366454124451, "peak_memory_bytes": 22206981120}
+{"step": 28, "generator_loss": 0.10136044770479202, "generator_grad_norm": 2.064396858215332, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.972245931625366, "peak_memory_bytes": 22206981120}
+{"step": 29, "generator_loss": 0.4598384201526642, "generator_grad_norm": 1.9268609285354614, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.113032817840576, "peak_memory_bytes": 22206981120}
+{"step": 30, "generator_loss": 0.19620059430599213, "generator_grad_norm": 2.3786990642547607, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07999920845032, "peak_memory_bytes": 22206981120}
+{"step": 31, "generator_loss": 0.5835046172142029, "generator_grad_norm": 2.161547899246216, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04690957069397, "peak_memory_bytes": 22206981120}
+{"step": 32, "generator_loss": 0.11053171008825302, "generator_grad_norm": 2.4489803314208984, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.2425742149353, "peak_memory_bytes": 22206981120}
+{"step": 33, "generator_loss": 0.2092645764350891, "generator_grad_norm": 2.077833652496338, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.21898627281189, "peak_memory_bytes": 22206981120}
+{"step": 34, "generator_loss": 0.4737488031387329, "generator_grad_norm": 2.04441237449646, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03843712806702, "peak_memory_bytes": 22206981120}
+{"step": 35, "generator_loss": 0.1503758430480957, "generator_grad_norm": 1.714919090270996, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18672800064087, "peak_memory_bytes": 22206981120}
+{"step": 36, "generator_loss": 0.29027894139289856, "generator_grad_norm": 2.060816526412964, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03187322616577, "peak_memory_bytes": 22206981120}
+{"step": 37, "generator_loss": 0.4373004734516144, "generator_grad_norm": 2.4680745601654053, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.274697065353394, "peak_memory_bytes": 22206981120}
+{"step": 38, "generator_loss": 0.20082363486289978, "generator_grad_norm": 1.8593153953552246, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00201678276062, "peak_memory_bytes": 22206981120}
+{"step": 39, "generator_loss": 0.1768437623977661, "generator_grad_norm": 2.1440670490264893, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.058372497558594, "peak_memory_bytes": 22206981120}
+{"step": 40, "generator_loss": 0.1985306739807129, "generator_grad_norm": 1.9369632005691528, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.125003814697266, "peak_memory_bytes": 22206981120}
+{"step": 41, "generator_loss": 0.26339566707611084, "generator_grad_norm": 2.1885290145874023, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98915123939514, "peak_memory_bytes": 22206981120}
+{"step": 42, "generator_loss": 0.2109604775905609, "generator_grad_norm": 1.8996261358261108, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11042070388794, "peak_memory_bytes": 22206981120}
+{"step": 43, "generator_loss": 0.094356968998909, "generator_grad_norm": 2.3804585933685303, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.149402141571045, "peak_memory_bytes": 22206981120}
+{"step": 44, "generator_loss": 0.27617213129997253, "generator_grad_norm": 2.0912458896636963, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.028812408447266, "peak_memory_bytes": 22206981120}
+{"step": 45, "generator_loss": 0.18252822756767273, "generator_grad_norm": 1.9014830589294434, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.184490442276, "peak_memory_bytes": 22206981120}
+{"step": 46, "generator_loss": 0.13081976771354675, "generator_grad_norm": 2.135986089706421, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.058429479599, "peak_memory_bytes": 22206981120}
+{"step": 47, "generator_loss": 0.1783914417028427, "generator_grad_norm": 2.172694444656372, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.269837617874146, "peak_memory_bytes": 22207118336}
+{"step": 48, "generator_loss": 0.21894024312496185, "generator_grad_norm": 1.5679658651351929, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02047276496887, "peak_memory_bytes": 22207118336}
+{"step": 49, "generator_loss": 0.2209823727607727, "generator_grad_norm": 1.6833468675613403, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12405967712402, "peak_memory_bytes": 22207118336}
+{"step": 50, "generator_loss": 0.23106001317501068, "generator_grad_norm": 2.092799663543701, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09538435935974, "peak_memory_bytes": 22207118336}
+{"step": 51, "generator_loss": 0.13642559945583344, "generator_grad_norm": 1.749455451965332, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98564910888672, "peak_memory_bytes": 22207118336}
+{"step": 52, "generator_loss": 0.19232206046581268, "generator_grad_norm": 2.308356285095215, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.114917516708374, "peak_memory_bytes": 22207118336}
+{"step": 53, "generator_loss": 0.22942402958869934, "generator_grad_norm": 1.9504988193511963, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1005003452301, "peak_memory_bytes": 22207118336}
+{"step": 54, "generator_loss": 0.10468857735395432, "generator_grad_norm": 2.3458995819091797, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96293830871582, "peak_memory_bytes": 22207118336}
+{"step": 55, "generator_loss": 0.20278865098953247, "generator_grad_norm": 1.7655377388000488, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12353587150574, "peak_memory_bytes": 22207118336}
+{"step": 56, "generator_loss": 0.3453182578086853, "generator_grad_norm": 2.082207679748535, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.998600244522095, "peak_memory_bytes": 22207118336}
+{"step": 57, "generator_loss": 0.10997936129570007, "generator_grad_norm": 1.8978182077407837, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1772255897522, "peak_memory_bytes": 22207118336}
+{"step": 58, "generator_loss": 0.1545736938714981, "generator_grad_norm": 1.7940751314163208, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04202604293823, "peak_memory_bytes": 22207118336}
+{"step": 59, "generator_loss": 0.20693515241146088, "generator_grad_norm": 1.6593823432922363, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.113507986068726, "peak_memory_bytes": 22207118336}
+{"step": 60, "generator_loss": 0.46453458070755005, "generator_grad_norm": 2.1202898025512695, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20208930969238, "peak_memory_bytes": 22207118336}
+{"step": 61, "generator_loss": 0.1649703085422516, "generator_grad_norm": 2.1613709926605225, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.956018686294556, "peak_memory_bytes": 22207118336}
+{"step": 62, "generator_loss": 0.5051997303962708, "generator_grad_norm": 2.0954551696777344, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14129328727722, "peak_memory_bytes": 22207118336}
+{"step": 63, "generator_loss": 0.2711363434791565, "generator_grad_norm": 1.6781401634216309, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12708139419556, "peak_memory_bytes": 22207118336}
+{"step": 64, "generator_loss": 0.21510738134384155, "generator_grad_norm": 2.0161614418029785, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02428364753723, "peak_memory_bytes": 22207118336}
+{"step": 65, "generator_loss": 0.07921797037124634, "generator_grad_norm": 1.723997950553894, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15039563179016, "peak_memory_bytes": 22207118336}
+{"step": 66, "generator_loss": 0.11020256578922272, "generator_grad_norm": 1.5533599853515625, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99210166931152, "peak_memory_bytes": 22207118336}
+{"step": 67, "generator_loss": 0.10587035119533539, "generator_grad_norm": 2.0068507194519043, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.115649461746216, "peak_memory_bytes": 22207118336}
+{"step": 68, "generator_loss": 0.45250821113586426, "generator_grad_norm": 1.8921374082565308, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9640212059021, "peak_memory_bytes": 22207118336}
+{"step": 69, "generator_loss": 0.27824828028678894, "generator_grad_norm": 2.373800039291382, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09378218650818, "peak_memory_bytes": 22207118336}
+{"step": 70, "generator_loss": 0.15435008704662323, "generator_grad_norm": 1.7820450067520142, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.05429434776306, "peak_memory_bytes": 22207118336}
+{"step": 71, "generator_loss": 0.16790249943733215, "generator_grad_norm": 2.016589403152466, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.044419288635254, "peak_memory_bytes": 22207118336}
+{"step": 72, "generator_loss": 0.18019677698612213, "generator_grad_norm": 1.443899393081665, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1490581035614, "peak_memory_bytes": 22207118336}
+{"step": 73, "generator_loss": 0.5693690180778503, "generator_grad_norm": 1.9536571502685547, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07878541946411, "peak_memory_bytes": 22207118336}
+{"step": 74, "generator_loss": 0.40548044443130493, "generator_grad_norm": 1.687199592590332, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.900065660476685, "peak_memory_bytes": 22207118336}
+{"step": 75, "generator_loss": 0.39971569180488586, "generator_grad_norm": 1.8163206577301025, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.06950902938843, "peak_memory_bytes": 22207118336}
+{"step": 76, "generator_loss": 0.14739258587360382, "generator_grad_norm": 1.6218496561050415, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95773124694824, "peak_memory_bytes": 22207118336}
+{"step": 77, "generator_loss": 0.26764172315597534, "generator_grad_norm": 1.9914854764938354, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12300109863281, "peak_memory_bytes": 22207118336}
+{"step": 78, "generator_loss": 0.3644295334815979, "generator_grad_norm": 1.904644250869751, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.92622423171997, "peak_memory_bytes": 22207118336}
+{"step": 79, "generator_loss": 0.21102826297283173, "generator_grad_norm": 1.9092063903808594, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.117005348205566, "peak_memory_bytes": 22207118336}
+{"step": 80, "generator_loss": 0.3131507337093353, "generator_grad_norm": 1.8351157903671265, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.05265259742737, "peak_memory_bytes": 22207118336}
+{"step": 81, "generator_loss": 0.12715329229831696, "generator_grad_norm": 1.1026030778884888, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0489935874939, "peak_memory_bytes": 22207118336}
+{"step": 82, "generator_loss": 0.286072313785553, "generator_grad_norm": 1.9211540222167969, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09076499938965, "peak_memory_bytes": 22207118336}
+{"step": 83, "generator_loss": 0.12720155715942383, "generator_grad_norm": 1.5156117677688599, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.114222288131714, "peak_memory_bytes": 22207118336}
+{"step": 84, "generator_loss": 0.6056622266769409, "generator_grad_norm": 1.8117142915725708, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04478859901428, "peak_memory_bytes": 22207118336}
+{"step": 85, "generator_loss": 0.12675894796848297, "generator_grad_norm": 2.1161346435546875, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15325903892517, "peak_memory_bytes": 22207118336}
+{"step": 86, "generator_loss": 0.05427943170070648, "generator_grad_norm": 1.6928939819335938, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99933171272278, "peak_memory_bytes": 22207118336}
+{"step": 87, "generator_loss": 0.20486155152320862, "generator_grad_norm": 1.9800868034362793, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15773415565491, "peak_memory_bytes": 22207118336}
+{"step": 88, "generator_loss": 0.12747497856616974, "generator_grad_norm": 1.7359751462936401, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9607675075531, "peak_memory_bytes": 22207118336}
+{"step": 89, "generator_loss": 0.5124704241752625, "generator_grad_norm": 1.6218199729919434, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18815851211548, "peak_memory_bytes": 22207118336}
+{"step": 90, "generator_loss": 0.34770143032073975, "generator_grad_norm": 1.5139350891113281, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18666052818298, "peak_memory_bytes": 22207118336}
+{"step": 91, "generator_loss": 0.14166973531246185, "generator_grad_norm": 1.4772366285324097, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98178219795227, "peak_memory_bytes": 22207118336}
+{"step": 92, "generator_loss": 0.09116818755865097, "generator_grad_norm": 2.1117913722991943, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11729145050049, "peak_memory_bytes": 22207118336}
+{"step": 93, "generator_loss": 0.09987238794565201, "generator_grad_norm": 1.069182276725769, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.13081502914429, "peak_memory_bytes": 22207118336}
+{"step": 94, "generator_loss": 0.4703330993652344, "generator_grad_norm": 1.6225694417953491, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98848009109497, "peak_memory_bytes": 22207118336}
+{"step": 95, "generator_loss": 0.06501450389623642, "generator_grad_norm": 1.577457308769226, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.097612142562866, "peak_memory_bytes": 22207118336}
+{"step": 96, "generator_loss": 0.5122780799865723, "generator_grad_norm": 1.711940884590149, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.063735008239746, "peak_memory_bytes": 22207118336}
+{"step": 97, "generator_loss": 0.44193294644355774, "generator_grad_norm": 1.372193694114685, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.152764081954956, "peak_memory_bytes": 22207118336}
+{"step": 98, "generator_loss": 0.21266871690750122, "generator_grad_norm": 2.3570799827575684, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00183057785034, "peak_memory_bytes": 22207118336}
+{"step": 99, "generator_loss": 0.1252344250679016, "generator_grad_norm": 2.306887626647949, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16262459754944, "peak_memory_bytes": 22207118336}
+{"step": 100, "generator_loss": 0.19680248200893402, "generator_grad_norm": 1.7149602174758911, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12523031234741, "peak_memory_bytes": 22207118336}
+{"step": 101, "generator_loss": 0.2786998450756073, "generator_grad_norm": 2.39565372467041, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.51030087471008, "peak_memory_bytes": 22207118336}
+{"step": 102, "generator_loss": 0.169748455286026, "generator_grad_norm": 1.2471272945404053, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11774039268494, "peak_memory_bytes": 22207118336}
+{"step": 103, "generator_loss": 0.11542259156703949, "generator_grad_norm": 1.2506163120269775, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99594211578369, "peak_memory_bytes": 22207118336}
+{"step": 104, "generator_loss": 0.549582302570343, "generator_grad_norm": 2.1666455268859863, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.157944202423096, "peak_memory_bytes": 22207118336}
+{"step": 105, "generator_loss": 0.07991378009319305, "generator_grad_norm": 1.5525431632995605, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98469114303589, "peak_memory_bytes": 22207118336}
+{"step": 106, "generator_loss": 0.0770406424999237, "generator_grad_norm": 2.3218228816986084, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.124536991119385, "peak_memory_bytes": 22207118336}
+{"step": 107, "generator_loss": 0.09948509186506271, "generator_grad_norm": 1.2758119106292725, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09735703468323, "peak_memory_bytes": 22207118336}
+{"step": 108, "generator_loss": 0.2551920711994171, "generator_grad_norm": 1.9859570264816284, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97001266479492, "peak_memory_bytes": 22207118336}
+{"step": 109, "generator_loss": 0.6636149883270264, "generator_grad_norm": 2.1065564155578613, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03523659706116, "peak_memory_bytes": 22207118336}
+{"step": 110, "generator_loss": 0.09088703244924545, "generator_grad_norm": 1.8359969854354858, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98190379142761, "peak_memory_bytes": 22207118336}
+{"step": 111, "generator_loss": 0.31744298338890076, "generator_grad_norm": 1.528120756149292, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12297177314758, "peak_memory_bytes": 22207118336}
+{"step": 112, "generator_loss": 0.3654513657093048, "generator_grad_norm": 2.223571300506592, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17362570762634, "peak_memory_bytes": 22207118336}
+{"step": 113, "generator_loss": 0.3399447202682495, "generator_grad_norm": 1.6721115112304688, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95553731918335, "peak_memory_bytes": 22207118336}
+{"step": 114, "generator_loss": 0.18649156391620636, "generator_grad_norm": 1.830312967300415, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.202714681625366, "peak_memory_bytes": 22207118336}
+{"step": 115, "generator_loss": 0.12383472919464111, "generator_grad_norm": 1.6206754446029663, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.968750953674316, "peak_memory_bytes": 22207118336}
+{"step": 116, "generator_loss": 0.09826242178678513, "generator_grad_norm": 1.2282404899597168, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10344743728638, "peak_memory_bytes": 22207118336}
+{"step": 117, "generator_loss": 0.3974589407444, "generator_grad_norm": 1.5881165266036987, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.90569591522217, "peak_memory_bytes": 22207118336}
+{"step": 118, "generator_loss": 0.06591197848320007, "generator_grad_norm": 1.3233375549316406, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.061739444732666, "peak_memory_bytes": 22207118336}
+{"step": 119, "generator_loss": 0.16710764169692993, "generator_grad_norm": 1.0506508350372314, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10111403465271, "peak_memory_bytes": 22207118336}
+{"step": 120, "generator_loss": 0.32538214325904846, "generator_grad_norm": 1.5325499773025513, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99561166763306, "peak_memory_bytes": 22207118336}
+{"step": 121, "generator_loss": 0.1709613800048828, "generator_grad_norm": 1.6564037799835205, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20836639404297, "peak_memory_bytes": 22207118336}
+{"step": 122, "generator_loss": 0.09711315482854843, "generator_grad_norm": 1.493303894996643, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96633720397949, "peak_memory_bytes": 22207118336}
+{"step": 123, "generator_loss": 0.2170570343732834, "generator_grad_norm": 1.9406495094299316, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17006731033325, "peak_memory_bytes": 22207118336}
+{"step": 124, "generator_loss": 0.11210376024246216, "generator_grad_norm": 1.531108021736145, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9752733707428, "peak_memory_bytes": 22207118336}
+{"step": 125, "generator_loss": 0.28281235694885254, "generator_grad_norm": 1.7190598249435425, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.138227462768555, "peak_memory_bytes": 22207118336}
+{"step": 126, "generator_loss": 0.44289857149124146, "generator_grad_norm": 1.367164969444275, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.110129833221436, "peak_memory_bytes": 22207118336}
+{"step": 127, "generator_loss": 0.20413579046726227, "generator_grad_norm": 0.9972870945930481, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94101524353027, "peak_memory_bytes": 22207118336}
+{"step": 128, "generator_loss": 0.13794344663619995, "generator_grad_norm": 1.3725943565368652, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07992959022522, "peak_memory_bytes": 22207118336}
+{"step": 129, "generator_loss": 0.3189041018486023, "generator_grad_norm": 1.855718970298767, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.93353080749512, "peak_memory_bytes": 22207118336}
+{"step": 130, "generator_loss": 0.14168894290924072, "generator_grad_norm": 1.5790796279907227, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1347439289093, "peak_memory_bytes": 22207118336}
+{"step": 131, "generator_loss": 0.3677601218223572, "generator_grad_norm": 1.7453455924987793, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0287561416626, "peak_memory_bytes": 22207118336}
+{"step": 132, "generator_loss": 0.19775670766830444, "generator_grad_norm": 1.389307975769043, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.151772022247314, "peak_memory_bytes": 22207118336}
+{"step": 133, "generator_loss": 0.36153823137283325, "generator_grad_norm": 2.027763843536377, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10840559005737, "peak_memory_bytes": 22207118336}
+{"step": 134, "generator_loss": 0.28720715641975403, "generator_grad_norm": 1.955604910850525, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.110820293426514, "peak_memory_bytes": 22207118336}
+{"step": 135, "generator_loss": 0.0777556449174881, "generator_grad_norm": 1.2539374828338623, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10799860954285, "peak_memory_bytes": 22207118336}
+{"step": 136, "generator_loss": 0.19153304398059845, "generator_grad_norm": 1.6876287460327148, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.945297956466675, "peak_memory_bytes": 22207118336}
+{"step": 137, "generator_loss": 0.28437426686286926, "generator_grad_norm": 2.0540761947631836, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.052649974823, "peak_memory_bytes": 22207118336}
+{"step": 138, "generator_loss": 0.29153677821159363, "generator_grad_norm": 1.2673789262771606, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.93187928199768, "peak_memory_bytes": 22207118336}
+{"step": 139, "generator_loss": 0.116434246301651, "generator_grad_norm": 1.2950834035873413, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11632513999939, "peak_memory_bytes": 22207118336}
+{"step": 140, "generator_loss": 0.07979385554790497, "generator_grad_norm": 1.419625163078308, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.095096588134766, "peak_memory_bytes": 22207118336}
+{"step": 141, "generator_loss": 0.19638139009475708, "generator_grad_norm": 2.178252935409546, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97245931625366, "peak_memory_bytes": 22207118336}
+{"step": 142, "generator_loss": 0.4955289959907532, "generator_grad_norm": 1.2298610210418701, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03717088699341, "peak_memory_bytes": 22207118336}
+{"step": 143, "generator_loss": 0.06325501203536987, "generator_grad_norm": 1.7509955167770386, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02950620651245, "peak_memory_bytes": 22207118336}
+{"step": 144, "generator_loss": 0.058600522577762604, "generator_grad_norm": 2.3572421073913574, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12733554840088, "peak_memory_bytes": 22207118336}
+{"step": 145, "generator_loss": 0.16697269678115845, "generator_grad_norm": 1.4516750574111938, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.970475912094116, "peak_memory_bytes": 22207118336}
+{"step": 146, "generator_loss": 0.04920510575175285, "generator_grad_norm": 1.5491533279418945, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15409755706787, "peak_memory_bytes": 22207118336}
+{"step": 147, "generator_loss": 0.27673494815826416, "generator_grad_norm": 1.2139939069747925, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.073166847229004, "peak_memory_bytes": 22207118336}
+{"step": 148, "generator_loss": 0.12903347611427307, "generator_grad_norm": 1.527613878250122, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00750470161438, "peak_memory_bytes": 22207118336}
+{"step": 149, "generator_loss": 0.11807131767272949, "generator_grad_norm": 1.7055776119232178, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11627531051636, "peak_memory_bytes": 22207118336}
+{"step": 150, "generator_loss": 0.1864510476589203, "generator_grad_norm": 1.4713269472122192, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99451184272766, "peak_memory_bytes": 22207118336}
+{"step": 151, "generator_loss": 0.08747143298387527, "generator_grad_norm": 1.7171798944473267, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1091845035553, "peak_memory_bytes": 22207118336}
+{"step": 152, "generator_loss": 0.46293309330940247, "generator_grad_norm": 1.5297632217407227, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97076940536499, "peak_memory_bytes": 22207118336}
+{"step": 153, "generator_loss": 0.10276111215353012, "generator_grad_norm": 1.0760835409164429, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.08268618583679, "peak_memory_bytes": 22207118336}
+{"step": 154, "generator_loss": 0.05859535187482834, "generator_grad_norm": 1.3634010553359985, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14302611351013, "peak_memory_bytes": 22207118336}
+{"step": 155, "generator_loss": 0.10235022753477097, "generator_grad_norm": 2.2332489490509033, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97030258178711, "peak_memory_bytes": 22207118336}
+{"step": 156, "generator_loss": 0.12837345898151398, "generator_grad_norm": 1.9165023565292358, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18291473388672, "peak_memory_bytes": 22207118336}
+{"step": 157, "generator_loss": 0.1372484564781189, "generator_grad_norm": 1.4935359954833984, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.096144676208496, "peak_memory_bytes": 22207118336}
+{"step": 158, "generator_loss": 0.10979568958282471, "generator_grad_norm": 1.3391016721725464, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20753026008606, "peak_memory_bytes": 22207118336}
+{"step": 159, "generator_loss": 0.1182255744934082, "generator_grad_norm": 1.3349103927612305, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.000701904296875, "peak_memory_bytes": 22207118336}
+{"step": 160, "generator_loss": 0.06709519028663635, "generator_grad_norm": 1.2584384679794312, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07549786567688, "peak_memory_bytes": 22207118336}
+{"step": 161, "generator_loss": 0.12811759114265442, "generator_grad_norm": 1.4303687810897827, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17150902748108, "peak_memory_bytes": 22207118336}
+{"step": 162, "generator_loss": 0.18193581700325012, "generator_grad_norm": 1.1127703189849854, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0437593460083, "peak_memory_bytes": 22207118336}
+{"step": 163, "generator_loss": 0.11109939217567444, "generator_grad_norm": 0.9563555121421814, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12169671058655, "peak_memory_bytes": 22207118336}
+{"step": 164, "generator_loss": 0.10705547034740448, "generator_grad_norm": 1.0621247291564941, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00697612762451, "peak_memory_bytes": 22207118336}
+{"step": 165, "generator_loss": 0.09903625398874283, "generator_grad_norm": 1.433396339416504, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.993234157562256, "peak_memory_bytes": 22207118336}
+{"step": 166, "generator_loss": 0.08583971112966537, "generator_grad_norm": 1.5489646196365356, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04107332229614, "peak_memory_bytes": 22207118336}
+{"step": 167, "generator_loss": 0.19312159717082977, "generator_grad_norm": 1.2218815088272095, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.094396114349365, "peak_memory_bytes": 22207118336}
+{"step": 168, "generator_loss": 0.03943687304854393, "generator_grad_norm": 1.4709060192108154, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16129755973816, "peak_memory_bytes": 22207118336}
+{"step": 169, "generator_loss": 0.2614232003688812, "generator_grad_norm": 1.462252140045166, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07854342460632, "peak_memory_bytes": 22207118336}
+{"step": 170, "generator_loss": 0.134031280875206, "generator_grad_norm": 1.1999953985214233, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.24088168144226, "peak_memory_bytes": 22207118336}
+{"step": 171, "generator_loss": 0.20547889173030853, "generator_grad_norm": 1.227653980255127, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.06246066093445, "peak_memory_bytes": 22207118336}
+{"step": 172, "generator_loss": 0.07037387788295746, "generator_grad_norm": 1.4282957315444946, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15812158584595, "peak_memory_bytes": 22207118336}
+{"step": 173, "generator_loss": 0.3590153157711029, "generator_grad_norm": 1.276779294013977, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98605561256409, "peak_memory_bytes": 22207118336}
+{"step": 174, "generator_loss": 0.11954743415117264, "generator_grad_norm": 0.9894145727157593, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1088502407074, "peak_memory_bytes": 22207118336}
+{"step": 175, "generator_loss": 0.21536748111248016, "generator_grad_norm": 1.348247766494751, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.06708812713623, "peak_memory_bytes": 22207118336}
+{"step": 176, "generator_loss": 0.11523948609828949, "generator_grad_norm": 1.7002099752426147, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96314358711243, "peak_memory_bytes": 22207118336}
+{"step": 177, "generator_loss": 0.2550477981567383, "generator_grad_norm": 1.3100453615188599, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1097526550293, "peak_memory_bytes": 22207118336}
+{"step": 178, "generator_loss": 0.5523772835731506, "generator_grad_norm": 1.694700837135315, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04629063606262, "peak_memory_bytes": 22207118336}
+{"step": 179, "generator_loss": 0.09759549051523209, "generator_grad_norm": 1.0119149684906006, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.19397449493408, "peak_memory_bytes": 22207118336}
+{"step": 180, "generator_loss": 0.11423351615667343, "generator_grad_norm": 1.476002812385559, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.135666370391846, "peak_memory_bytes": 22207118336}
+{"step": 181, "generator_loss": 0.07288438826799393, "generator_grad_norm": 1.3816279172897339, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0425660610199, "peak_memory_bytes": 22207118336}
+{"step": 182, "generator_loss": 0.37147441506385803, "generator_grad_norm": 2.2076661586761475, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.170247316360474, "peak_memory_bytes": 22207118336}
+{"step": 183, "generator_loss": 0.0893692672252655, "generator_grad_norm": 0.8229760527610779, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.089274168014526, "peak_memory_bytes": 22207118336}
+{"step": 184, "generator_loss": 0.1596575677394867, "generator_grad_norm": 1.8093706369400024, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14101004600525, "peak_memory_bytes": 22207118336}
+{"step": 185, "generator_loss": 0.181111678481102, "generator_grad_norm": 1.807491660118103, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02031397819519, "peak_memory_bytes": 22207118336}
+{"step": 186, "generator_loss": 0.1645023375749588, "generator_grad_norm": 1.2811927795410156, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.22061896324158, "peak_memory_bytes": 22207118336}
+{"step": 187, "generator_loss": 0.15172022581100464, "generator_grad_norm": 1.2527893781661987, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12616443634033, "peak_memory_bytes": 22207118336}
+{"step": 188, "generator_loss": 0.19916775822639465, "generator_grad_norm": 1.3435231447219849, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.005040407180786, "peak_memory_bytes": 22207118336}
+{"step": 189, "generator_loss": 0.16932426393032074, "generator_grad_norm": 1.3819657564163208, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.103402853012085, "peak_memory_bytes": 22207118336}
+{"step": 190, "generator_loss": 0.23398858308792114, "generator_grad_norm": 1.663132667541504, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.081393003463745, "peak_memory_bytes": 22207118336}
+{"step": 191, "generator_loss": 0.06716237962245941, "generator_grad_norm": 0.9540762305259705, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14878869056702, "peak_memory_bytes": 22207118336}
+{"step": 192, "generator_loss": 0.053515173494815826, "generator_grad_norm": 0.8989765048027039, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.26535964012146, "peak_memory_bytes": 22207118336}
+{"step": 193, "generator_loss": 0.28825223445892334, "generator_grad_norm": 1.822442650794983, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07545065879822, "peak_memory_bytes": 22207118336}
+{"step": 194, "generator_loss": 0.25262895226478577, "generator_grad_norm": 1.7374378442764282, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18030309677124, "peak_memory_bytes": 22207118336}
+{"step": 195, "generator_loss": 0.10312087088823318, "generator_grad_norm": 1.296190619468689, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.043193340301514, "peak_memory_bytes": 22207118336}
+{"step": 196, "generator_loss": 0.1158909946680069, "generator_grad_norm": 0.978812038898468, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.21153497695923, "peak_memory_bytes": 22207118336}
+{"step": 197, "generator_loss": 0.2739603519439697, "generator_grad_norm": 1.3645747900009155, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.996763467788696, "peak_memory_bytes": 22207118336}
+{"step": 198, "generator_loss": 0.39362695813179016, "generator_grad_norm": 1.7177398204803467, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14187026023865, "peak_memory_bytes": 22207118336}
+{"step": 199, "generator_loss": 0.07802467048168182, "generator_grad_norm": 1.6359587907791138, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12679100036621, "peak_memory_bytes": 22207118336}
+{"step": 200, "generator_loss": 0.07862439751625061, "generator_grad_norm": 1.3144112825393677, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.986249685287476, "peak_memory_bytes": 22207118336}

outputs/temporal_nope/a4_tdrope_hard_lr1e7_probe200_seed20260805_kerrigan/validation_metrics.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "validation_loss": 0.2635482151526958, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 22.943557262420654, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 100, "validation_loss": 0.19042654358781874, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.483413219451904, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 200, "validation_loss": 0.1463205151958391, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.64357328414917, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}

outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt", "sha256": "9c73b32512466e1761498cc9b47616c7ad7e4ceb6a8fbf930bb9dffc4e51c348", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt", "sha256": "474a2d6d0e092c5eac9bd73b7dc6d9fae848f02ca555abcd425135269a764b55", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/resolved_config.yaml

@@ -0,0 +1,123 @@
+experiment:
+  id: a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: Hard Temporal DroPE
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: none
+  temporal_rope_alpha: 0.0
+  temporal_rope_schedule: constant
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 0
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+  target_visible_lengths:
+  - 21
+  - 42
+  - 84
+  train_length: 21
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 2.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/train.log

@@ -0,0 +1,259 @@
+W0806 19:54:01.441000 3024960 site-packages/torch/distributed/run.py:793] 
+W0806 19:54:01.441000 3024960 site-packages/torch/distributed/run.py:793] *****************************************
+W0806 19:54:01.441000 3024960 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0806 19:54:01.441000 3024960 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank4]:[W806 19:54:11.623620111 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank1]:[W806 19:54:11.642578112 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W806 19:54:12.676204611 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank2]:[W806 19:54:12.697532734 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W806 19:54:12.713233027 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank6]:[W806 19:54:12.722431614 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank7]:[W806 19:54:12.740894497 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank0]:[W806 19:54:12.972020512 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.26354822 alpha=0.000000 samples=32 seconds=22.619
+step=1 loss=0.75935781 grad_norm=2.223714 alpha=0.000000 seconds=33.482
+step=2 loss=0.60316056 grad_norm=2.933211 alpha=0.000000 seconds=33.598
+step=3 loss=0.35381573 grad_norm=2.790839 alpha=0.000000 seconds=33.865
+step=4 loss=0.44479188 grad_norm=2.668969 alpha=0.000000 seconds=33.862
+step=5 loss=0.13669759 grad_norm=2.161324 alpha=0.000000 seconds=34.032
+step=6 loss=0.14745352 grad_norm=2.103647 alpha=0.000000 seconds=33.888
+step=7 loss=0.55810076 grad_norm=2.754236 alpha=0.000000 seconds=34.045
+step=8 loss=0.30983847 grad_norm=2.175804 alpha=0.000000 seconds=34.041
+step=9 loss=0.19592808 grad_norm=2.446187 alpha=0.000000 seconds=33.944
+step=10 loss=0.72553855 grad_norm=2.596434 alpha=0.000000 seconds=34.023
+step=11 loss=0.19189744 grad_norm=2.342392 alpha=0.000000 seconds=33.893
+step=12 loss=0.22997066 grad_norm=1.950853 alpha=0.000000 seconds=34.040
+step=13 loss=0.52567893 grad_norm=2.587671 alpha=0.000000 seconds=34.048
+step=14 loss=0.58557266 grad_norm=2.340523 alpha=0.000000 seconds=33.897
+step=15 loss=0.07713816 grad_norm=2.549052 alpha=0.000000 seconds=34.116
+step=16 loss=0.14214294 grad_norm=1.819709 alpha=0.000000 seconds=33.969
+step=17 loss=0.43544453 grad_norm=2.384693 alpha=0.000000 seconds=34.059
+step=18 loss=0.27915406 grad_norm=2.205927 alpha=0.000000 seconds=33.894
+step=19 loss=0.17618456 grad_norm=2.133121 alpha=0.000000 seconds=34.008
+step=20 loss=0.40435466 grad_norm=1.821976 alpha=0.000000 seconds=34.018
+step=21 loss=0.13712730 grad_norm=2.359310 alpha=0.000000 seconds=33.910
+step=22 loss=0.47017407 grad_norm=2.419784 alpha=0.000000 seconds=34.000
+step=23 loss=0.33177173 grad_norm=2.247394 alpha=0.000000 seconds=33.960
+step=24 loss=0.26156193 grad_norm=1.852762 alpha=0.000000 seconds=33.799
+step=25 loss=0.09069069 grad_norm=1.790111 alpha=0.000000 seconds=33.950
+step=26 loss=0.22446437 grad_norm=1.668919 alpha=0.000000 seconds=33.853
+step=27 loss=0.30181167 grad_norm=1.707894 alpha=0.000000 seconds=33.967
+step=28 loss=0.09580465 grad_norm=1.813926 alpha=0.000000 seconds=33.844
+step=29 loss=0.41668805 grad_norm=1.558098 alpha=0.000000 seconds=33.977
+step=30 loss=0.17205344 grad_norm=2.158468 alpha=0.000000 seconds=33.970
+step=31 loss=0.54084814 grad_norm=1.978491 alpha=0.000000 seconds=33.772
+step=32 loss=0.09555051 grad_norm=2.078477 alpha=0.000000 seconds=33.989
+step=33 loss=0.18402731 grad_norm=2.052625 alpha=0.000000 seconds=33.931
+step=34 loss=0.44334984 grad_norm=1.879377 alpha=0.000000 seconds=33.771
+step=35 loss=0.13494986 grad_norm=1.615761 alpha=0.000000 seconds=33.879
+step=36 loss=0.26763722 grad_norm=1.817520 alpha=0.000000 seconds=33.852
+step=37 loss=0.39042383 grad_norm=2.255382 alpha=0.000000 seconds=33.925
+step=38 loss=0.17538355 grad_norm=1.825204 alpha=0.000000 seconds=33.737
+step=39 loss=0.14666024 grad_norm=1.660090 alpha=0.000000 seconds=33.836
+step=40 loss=0.16939025 grad_norm=1.823641 alpha=0.000000 seconds=33.896
+step=41 loss=0.23330966 grad_norm=1.784105 alpha=0.000000 seconds=33.769
+step=42 loss=0.18339731 grad_norm=1.569561 alpha=0.000000 seconds=33.962
+step=43 loss=0.07884911 grad_norm=2.113375 alpha=0.000000 seconds=33.913
+step=44 loss=0.23759742 grad_norm=1.644974 alpha=0.000000 seconds=33.757
+step=45 loss=0.16423596 grad_norm=1.502872 alpha=0.000000 seconds=33.881
+step=46 loss=0.11207359 grad_norm=1.990921 alpha=0.000000 seconds=33.740
+step=47 loss=0.14900078 grad_norm=1.817151 alpha=0.000000 seconds=33.890
+step=48 loss=0.19298272 grad_norm=1.210970 alpha=0.000000 seconds=33.758
+step=49 loss=0.18891580 grad_norm=1.628075 alpha=0.000000 seconds=33.895
+step=50 loss=0.19762142 grad_norm=1.790472 alpha=0.000000 seconds=33.863
+step=51 loss=0.10816649 grad_norm=1.496627 alpha=0.000000 seconds=33.761
+step=52 loss=0.15071020 grad_norm=2.249658 alpha=0.000000 seconds=33.926
+step=53 loss=0.19765823 grad_norm=1.793454 alpha=0.000000 seconds=33.922
+step=54 loss=0.08554039 grad_norm=2.183553 alpha=0.000000 seconds=33.765
+step=55 loss=0.17895278 grad_norm=1.671954 alpha=0.000000 seconds=33.862
+step=56 loss=0.29807442 grad_norm=1.833816 alpha=0.000000 seconds=33.718
+step=57 loss=0.09514660 grad_norm=1.905537 alpha=0.000000 seconds=33.856
+step=58 loss=0.13721737 grad_norm=1.440923 alpha=0.000000 seconds=33.706
+step=59 loss=0.18440065 grad_norm=1.363409 alpha=0.000000 seconds=33.817
+step=60 loss=0.39827412 grad_norm=1.935408 alpha=0.000000 seconds=33.843
+step=61 loss=0.14426254 grad_norm=1.874481 alpha=0.000000 seconds=33.783
+step=62 loss=0.43157205 grad_norm=1.861359 alpha=0.000000 seconds=33.873
+step=63 loss=0.22527032 grad_norm=1.183560 alpha=0.000000 seconds=33.850
+step=64 loss=0.19086581 grad_norm=1.791168 alpha=0.000000 seconds=33.705
+step=65 loss=0.06410085 grad_norm=1.517551 alpha=0.000000 seconds=33.826
+step=66 loss=0.08699367 grad_norm=1.445365 alpha=0.000000 seconds=33.673
+step=67 loss=0.07813928 grad_norm=1.633520 alpha=0.000000 seconds=33.793
+step=68 loss=0.39476985 grad_norm=1.697856 alpha=0.000000 seconds=33.665
+step=69 loss=0.21365517 grad_norm=2.271358 alpha=0.000000 seconds=33.790
+step=70 loss=0.12494264 grad_norm=1.652606 alpha=0.000000 seconds=33.803
+step=71 loss=0.12745367 grad_norm=1.949244 alpha=0.000000 seconds=33.705
+step=72 loss=0.14017223 grad_norm=1.091326 alpha=0.000000 seconds=33.866
+step=73 loss=0.47744799 grad_norm=1.863165 alpha=0.000000 seconds=33.827
+step=74 loss=0.32345542 grad_norm=1.526154 alpha=0.000000 seconds=33.652
+step=75 loss=0.32232893 grad_norm=1.577615 alpha=0.000000 seconds=33.835
+step=76 loss=0.10670803 grad_norm=1.253050 alpha=0.000000 seconds=33.664
+step=77 loss=0.20213534 grad_norm=1.775066 alpha=0.000000 seconds=33.849
+step=78 loss=0.27257690 grad_norm=1.301365 alpha=0.000000 seconds=33.673
+step=79 loss=0.16312356 grad_norm=1.496948 alpha=0.000000 seconds=33.792
+step=80 loss=0.24257010 grad_norm=1.706037 alpha=0.000000 seconds=33.859
+step=81 loss=0.11316301 grad_norm=0.751791 alpha=0.000000 seconds=33.706
+step=82 loss=0.19950388 grad_norm=1.198079 alpha=0.000000 seconds=33.854
+step=83 loss=0.10085530 grad_norm=1.148947 alpha=0.000000 seconds=33.851
+step=84 loss=0.47650483 grad_norm=1.553979 alpha=0.000000 seconds=33.668
+step=85 loss=0.09645943 grad_norm=1.511721 alpha=0.000000 seconds=33.820
+step=86 loss=0.04516522 grad_norm=1.120702 alpha=0.000000 seconds=33.643
+step=87 loss=0.15462098 grad_norm=1.460508 alpha=0.000000 seconds=33.806
+step=88 loss=0.11338810 grad_norm=1.404099 alpha=0.000000 seconds=33.665
+step=89 loss=0.43421811 grad_norm=1.228755 alpha=0.000000 seconds=33.772
+step=90 loss=0.26641542 grad_norm=1.464799 alpha=0.000000 seconds=33.767
+step=91 loss=0.11470888 grad_norm=1.230817 alpha=0.000000 seconds=33.635
+step=92 loss=0.07105367 grad_norm=1.617311 alpha=0.000000 seconds=33.811
+step=93 loss=0.07540740 grad_norm=0.679684 alpha=0.000000 seconds=33.786
+step=94 loss=0.40294635 grad_norm=1.232529 alpha=0.000000 seconds=33.660
+step=95 loss=0.05332119 grad_norm=1.191226 alpha=0.000000 seconds=33.807
+step=96 loss=0.44524965 grad_norm=1.164661 alpha=0.000000 seconds=33.669
+step=97 loss=0.36762464 grad_norm=0.794524 alpha=0.000000 seconds=33.794
+step=98 loss=0.16921875 grad_norm=1.883905 alpha=0.000000 seconds=33.637
+step=99 loss=0.10734279 grad_norm=1.729467 alpha=0.000000 seconds=33.747
+step=100 loss=0.15980434 grad_norm=1.121718 alpha=0.000000 seconds=33.776
+validation step=100 loss=0.14690994 alpha=0.000000 samples=32 seconds=23.254
+Start gathering distributed model states...
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states.../home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Start gathering distributed model states...
+/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt sha256=9c73b32512466e1761498cc9b47616c7ad7e4ceb6a8fbf930bb9dffc4e51c348
+step=101 loss=0.19841097 grad_norm=1.574268 alpha=0.000000 seconds=33.143
+step=102 loss=0.11785068 grad_norm=0.726850 alpha=0.000000 seconds=33.770
+step=103 loss=0.10634899 grad_norm=0.662060 alpha=0.000000 seconds=33.655
+step=104 loss=0.40984410 grad_norm=1.235051 alpha=0.000000 seconds=33.775
+step=105 loss=0.05807800 grad_norm=0.702543 alpha=0.000000 seconds=33.629
+step=106 loss=0.06164713 grad_norm=1.868279 alpha=0.000000 seconds=33.795
+step=107 loss=0.08294948 grad_norm=0.856796 alpha=0.000000 seconds=33.808
+step=108 loss=0.18574186 grad_norm=1.385939 alpha=0.000000 seconds=33.669
+step=109 loss=0.52388173 grad_norm=1.292452 alpha=0.000000 seconds=33.741
+step=110 loss=0.07563873 grad_norm=1.227535 alpha=0.000000 seconds=33.649
+step=111 loss=0.23264055 grad_norm=0.898203 alpha=0.000000 seconds=33.762
+step=112 loss=0.27945012 grad_norm=1.300601 alpha=0.000000 seconds=33.789
+step=113 loss=0.27725202 grad_norm=0.981235 alpha=0.000000 seconds=33.631
+step=114 loss=0.11895660 grad_norm=1.043160 alpha=0.000000 seconds=33.728
+step=115 loss=0.10890976 grad_norm=1.096831 alpha=0.000000 seconds=33.645
+step=116 loss=0.08057566 grad_norm=0.728835 alpha=0.000000 seconds=33.767
+step=117 loss=0.28552142 grad_norm=0.722207 alpha=0.000000 seconds=33.645
+step=118 loss=0.05057976 grad_norm=1.090677 alpha=0.000000 seconds=33.804
+step=119 loss=0.13696119 grad_norm=0.591582 alpha=0.000000 seconds=33.811
+step=120 loss=0.22802085 grad_norm=0.889178 alpha=0.000000 seconds=33.670
+step=121 loss=0.11770721 grad_norm=0.773860 alpha=0.000000 seconds=33.836
+step=122 loss=0.07326443 grad_norm=0.871488 alpha=0.000000 seconds=33.682
+step=123 loss=0.15939526 grad_norm=0.636248 alpha=0.000000 seconds=33.908
+step=124 loss=0.08880942 grad_norm=0.755056 alpha=0.000000 seconds=33.699
+step=125 loss=0.22726306 grad_norm=0.848602 alpha=0.000000 seconds=33.878
+step=126 loss=0.33075160 grad_norm=0.707132 alpha=0.000000 seconds=33.822
+step=127 loss=0.15914635 grad_norm=0.375934 alpha=0.000000 seconds=33.818
+step=128 loss=0.10291738 grad_norm=0.527633 alpha=0.000000 seconds=33.894
+step=129 loss=0.21067800 grad_norm=0.946491 alpha=0.000000 seconds=33.751
+step=130 loss=0.11703181 grad_norm=0.653545 alpha=0.000000 seconds=33.916
+step=131 loss=0.25263697 grad_norm=0.700023 alpha=0.000000 seconds=33.745
+step=132 loss=0.13378897 grad_norm=0.588160 alpha=0.000000 seconds=33.876
+step=133 loss=0.26069170 grad_norm=0.892882 alpha=0.000000 seconds=33.803
+step=134 loss=0.21071282 grad_norm=0.914561 alpha=0.000000 seconds=33.927
+step=135 loss=0.06307131 grad_norm=0.562107 alpha=0.000000 seconds=33.984
+step=136 loss=0.13477406 grad_norm=0.685682 alpha=0.000000 seconds=33.825
+step=137 loss=0.20405443 grad_norm=1.102290 alpha=0.000000 seconds=33.954
+step=138 loss=0.22960328 grad_norm=0.485345 alpha=0.000000 seconds=33.860
+step=139 loss=0.09174458 grad_norm=0.499848 alpha=0.000000 seconds=33.989
+step=140 loss=0.05428297 grad_norm=0.689067 alpha=0.000000 seconds=33.975
+step=141 loss=0.13453837 grad_norm=0.735649 alpha=0.000000 seconds=33.878
+step=142 loss=0.33643216 grad_norm=0.479218 alpha=0.000000 seconds=33.989
+step=143 loss=0.05421725 grad_norm=0.644185 alpha=0.000000 seconds=33.836
+step=144 loss=0.04892465 grad_norm=1.202953 alpha=0.000000 seconds=33.916
+step=145 loss=0.13452448 grad_norm=0.514850 alpha=0.000000 seconds=33.812
+step=146 loss=0.04065841 grad_norm=1.076193 alpha=0.000000 seconds=33.943
+step=147 loss=0.19051370 grad_norm=0.384166 alpha=0.000000 seconds=33.896
+step=148 loss=0.10900343 grad_norm=0.461516 alpha=0.000000 seconds=33.806
+step=149 loss=0.09254646 grad_norm=0.553590 alpha=0.000000 seconds=33.982
+step=150 loss=0.14974165 grad_norm=0.593260 alpha=0.000000 seconds=33.733
+step=151 loss=0.06853129 grad_norm=0.465885 alpha=0.000000 seconds=33.918
+step=152 loss=0.36141002 grad_norm=0.507393 alpha=0.000000 seconds=33.742
+step=153 loss=0.07388052 grad_norm=0.260216 alpha=0.000000 seconds=33.890
+step=154 loss=0.04485170 grad_norm=0.427207 alpha=0.000000 seconds=33.885
+step=155 loss=0.07534618 grad_norm=0.755936 alpha=0.000000 seconds=33.720
+step=156 loss=0.09686054 grad_norm=0.774386 alpha=0.000000 seconds=33.858
+step=157 loss=0.09431375 grad_norm=0.578817 alpha=0.000000 seconds=33.720
+step=158 loss=0.09127193 grad_norm=0.464940 alpha=0.000000 seconds=33.849
+step=159 loss=0.08054102 grad_norm=0.338110 alpha=0.000000 seconds=33.712
+step=160 loss=0.05224596 grad_norm=0.366043 alpha=0.000000 seconds=33.850
+step=161 loss=0.09968896 grad_norm=0.654179 alpha=0.000000 seconds=33.900
+step=162 loss=0.13927594 grad_norm=0.334574 alpha=0.000000 seconds=33.772
+step=163 loss=0.08443568 grad_norm=0.278962 alpha=0.000000 seconds=33.921
+step=164 loss=0.07604159 grad_norm=0.347732 alpha=0.000000 seconds=33.766
+step=165 loss=0.09156201 grad_norm=0.475819 alpha=0.000000 seconds=33.797
+step=166 loss=0.06923417 grad_norm=0.656102 alpha=0.000000 seconds=33.725
+step=167 loss=0.13126005 grad_norm=0.582200 alpha=0.000000 seconds=33.841
+step=168 loss=0.03211470 grad_norm=0.494465 alpha=0.000000 seconds=33.870
+step=169 loss=0.18781652 grad_norm=0.532820 alpha=0.000000 seconds=33.683
+step=170 loss=0.11606246 grad_norm=0.439514 alpha=0.000000 seconds=33.788
+step=171 loss=0.14174671 grad_norm=0.300421 alpha=0.000000 seconds=33.712
+step=172 loss=0.05593271 grad_norm=0.483740 alpha=0.000000 seconds=33.808
+step=173 loss=0.26347318 grad_norm=0.437653 alpha=0.000000 seconds=33.698
+step=174 loss=0.08855049 grad_norm=0.345462 alpha=0.000000 seconds=33.802
+step=175 loss=0.17608048 grad_norm=0.661532 alpha=0.000000 seconds=33.772
+step=176 loss=0.09613776 grad_norm=0.582581 alpha=0.000000 seconds=33.682
+step=177 loss=0.18174821 grad_norm=0.487361 alpha=0.000000 seconds=33.798
+step=178 loss=0.38082069 grad_norm=0.681739 alpha=0.000000 seconds=33.676
+step=179 loss=0.07860675 grad_norm=0.366972 alpha=0.000000 seconds=33.810
+step=180 loss=0.08565001 grad_norm=0.546648 alpha=0.000000 seconds=33.767
+step=181 loss=0.05822566 grad_norm=0.404194 alpha=0.000000 seconds=33.661
+step=182 loss=0.26365465 grad_norm=0.623132 alpha=0.000000 seconds=33.834
+step=183 loss=0.07049100 grad_norm=0.228588 alpha=0.000000 seconds=33.656
+step=184 loss=0.11076851 grad_norm=0.528344 alpha=0.000000 seconds=33.761
+step=185 loss=0.12941755 grad_norm=0.591120 alpha=0.000000 seconds=33.640
+step=186 loss=0.11945979 grad_norm=0.446185 alpha=0.000000 seconds=33.830
+step=187 loss=0.11358052 grad_norm=0.336871 alpha=0.000000 seconds=33.852
+step=188 loss=0.13648066 grad_norm=0.399636 alpha=0.000000 seconds=33.695
+step=189 loss=0.13921918 grad_norm=0.686332 alpha=0.000000 seconds=33.819
+step=190 loss=0.17315605 grad_norm=0.624339 alpha=0.000000 seconds=33.698
+step=191 loss=0.05595165 grad_norm=0.307320 alpha=0.000000 seconds=33.851
+step=192 loss=0.04315925 grad_norm=0.317438 alpha=0.000000 seconds=33.834
+step=193 loss=0.19792590 grad_norm=0.683195 alpha=0.000000 seconds=33.691
+step=194 loss=0.19321944 grad_norm=0.549833 alpha=0.000000 seconds=33.782
+step=195 loss=0.09127001 grad_norm=0.410555 alpha=0.000000 seconds=33.642
+step=196 loss=0.08986714 grad_norm=0.329658 alpha=0.000000 seconds=33.861
+step=197 loss=0.19208924 grad_norm=0.432852 alpha=0.000000 seconds=33.661
+step=198 loss=0.29705456 grad_norm=0.753168 alpha=0.000000 seconds=33.903
+step=199 loss=0.05508996 grad_norm=0.480704 alpha=0.000000 seconds=33.866
+step=200 loss=0.06518029 grad_norm=0.470366 alpha=0.000000 seconds=33.661
+validation step=200 loss=0.11282415 alpha=0.000000 samples=32 seconds=23.329
+Start gathering distributed model states...
+Start gathering distributed model states...
+Start gathering distributed model states...Start gathering distributed model states...
+
+Start gathering distributed model states...
+Start gathering distributed model states...Start gathering distributed model states...
+
+Start gathering distributed model states...
+Model saved to /home/jiaqi/NoPE/outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt sha256=474a2d6d0e092c5eac9bd73b7dc6d9fae848f02ca555abcd425135269a764b55

outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/training_metrics.jsonl

@@ -0,0 +1,200 @@
+{"step": 1, "generator_loss": 0.7593578100204468, "generator_grad_norm": 2.2237138748168945, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.481818437576294, "peak_memory_bytes": 20785906176}
+{"step": 2, "generator_loss": 0.603160560131073, "generator_grad_norm": 2.933210611343384, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.597740173339844, "peak_memory_bytes": 22206258176}
+{"step": 3, "generator_loss": 0.3538157343864441, "generator_grad_norm": 2.790839195251465, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86470651626587, "peak_memory_bytes": 22206258176}
+{"step": 4, "generator_loss": 0.44479188323020935, "generator_grad_norm": 2.668968915939331, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86240267753601, "peak_memory_bytes": 22206258176}
+{"step": 5, "generator_loss": 0.13669759035110474, "generator_grad_norm": 2.1613235473632812, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03248047828674, "peak_memory_bytes": 22206694400}
+{"step": 6, "generator_loss": 0.14745351672172546, "generator_grad_norm": 2.103646993637085, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.88751029968262, "peak_memory_bytes": 22206694400}
+{"step": 7, "generator_loss": 0.5581007599830627, "generator_grad_norm": 2.7542359828948975, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04511642456055, "peak_memory_bytes": 22206694400}
+{"step": 8, "generator_loss": 0.3098384737968445, "generator_grad_norm": 2.1758041381835938, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04117703437805, "peak_memory_bytes": 22206694400}
+{"step": 9, "generator_loss": 0.19592808187007904, "generator_grad_norm": 2.4461865425109863, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.944416522979736, "peak_memory_bytes": 22206694400}
+{"step": 10, "generator_loss": 0.7255385518074036, "generator_grad_norm": 2.5964341163635254, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.022608518600464, "peak_memory_bytes": 22206694400}
+{"step": 11, "generator_loss": 0.1918974369764328, "generator_grad_norm": 2.3423919677734375, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.892624855041504, "peak_memory_bytes": 22206694400}
+{"step": 12, "generator_loss": 0.22997066378593445, "generator_grad_norm": 1.9508527517318726, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04012489318848, "peak_memory_bytes": 22206694400}
+{"step": 13, "generator_loss": 0.5256789326667786, "generator_grad_norm": 2.5876708030700684, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.048203229904175, "peak_memory_bytes": 22206694400}
+{"step": 14, "generator_loss": 0.5855726599693298, "generator_grad_norm": 2.340522527694702, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.89723610877991, "peak_memory_bytes": 22206694400}
+{"step": 15, "generator_loss": 0.07713816314935684, "generator_grad_norm": 2.549051523208618, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11636447906494, "peak_memory_bytes": 22206694400}
+{"step": 16, "generator_loss": 0.14214293658733368, "generator_grad_norm": 1.8197094202041626, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.969231367111206, "peak_memory_bytes": 22206694400}
+{"step": 17, "generator_loss": 0.43544453382492065, "generator_grad_norm": 2.384692907333374, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.05929183959961, "peak_memory_bytes": 22206694400}
+{"step": 18, "generator_loss": 0.27915406227111816, "generator_grad_norm": 2.2059273719787598, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.89400863647461, "peak_memory_bytes": 22206694400}
+{"step": 19, "generator_loss": 0.17618456482887268, "generator_grad_norm": 2.1331210136413574, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00800895690918, "peak_memory_bytes": 22206694400}
+{"step": 20, "generator_loss": 0.40435466170310974, "generator_grad_norm": 1.8219761848449707, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0175039768219, "peak_memory_bytes": 22206831616}
+{"step": 21, "generator_loss": 0.13712729513645172, "generator_grad_norm": 2.3593099117279053, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.910184383392334, "peak_memory_bytes": 22206831616}
+{"step": 22, "generator_loss": 0.47017407417297363, "generator_grad_norm": 2.419783592224121, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00046348571777, "peak_memory_bytes": 22206831616}
+{"step": 23, "generator_loss": 0.33177173137664795, "generator_grad_norm": 2.247394323348999, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96047592163086, "peak_memory_bytes": 22206981120}
+{"step": 24, "generator_loss": 0.26156193017959595, "generator_grad_norm": 1.85276198387146, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.798949003219604, "peak_memory_bytes": 22206981120}
+{"step": 25, "generator_loss": 0.09069069474935532, "generator_grad_norm": 1.7901114225387573, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95024394989014, "peak_memory_bytes": 22206981120}
+{"step": 26, "generator_loss": 0.22446437180042267, "generator_grad_norm": 1.6689187288284302, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.853331089019775, "peak_memory_bytes": 22206981120}
+{"step": 27, "generator_loss": 0.30181166529655457, "generator_grad_norm": 1.7078937292099, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96665692329407, "peak_memory_bytes": 22206981120}
+{"step": 28, "generator_loss": 0.09580464661121368, "generator_grad_norm": 1.8139264583587646, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.84361791610718, "peak_memory_bytes": 22206981120}
+{"step": 29, "generator_loss": 0.41668805480003357, "generator_grad_norm": 1.5580980777740479, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.976526498794556, "peak_memory_bytes": 22206981120}
+{"step": 30, "generator_loss": 0.17205344140529633, "generator_grad_norm": 2.158468246459961, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.969921827316284, "peak_memory_bytes": 22206981120}
+{"step": 31, "generator_loss": 0.5408481359481812, "generator_grad_norm": 1.978490948677063, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.77200675010681, "peak_memory_bytes": 22206981120}
+{"step": 32, "generator_loss": 0.09555050730705261, "generator_grad_norm": 2.078477144241333, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98862028121948, "peak_memory_bytes": 22206981120}
+{"step": 33, "generator_loss": 0.1840273141860962, "generator_grad_norm": 2.0526251792907715, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.93055510520935, "peak_memory_bytes": 22206981120}
+{"step": 34, "generator_loss": 0.44334983825683594, "generator_grad_norm": 1.8793773651123047, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.77054262161255, "peak_memory_bytes": 22206981120}
+{"step": 35, "generator_loss": 0.13494986295700073, "generator_grad_norm": 1.6157608032226562, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.87887668609619, "peak_memory_bytes": 22206981120}
+{"step": 36, "generator_loss": 0.2676372230052948, "generator_grad_norm": 1.8175199031829834, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85161900520325, "peak_memory_bytes": 22206981120}
+{"step": 37, "generator_loss": 0.39042383432388306, "generator_grad_norm": 2.2553818225860596, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.92475724220276, "peak_memory_bytes": 22206981120}
+{"step": 38, "generator_loss": 0.1753835529088974, "generator_grad_norm": 1.8252044916152954, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.73708391189575, "peak_memory_bytes": 22206981120}
+{"step": 39, "generator_loss": 0.1466602385044098, "generator_grad_norm": 1.6600898504257202, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.83643054962158, "peak_memory_bytes": 22206981120}
+{"step": 40, "generator_loss": 0.1693902462720871, "generator_grad_norm": 1.823641061782837, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.8963258266449, "peak_memory_bytes": 22206981120}
+{"step": 41, "generator_loss": 0.23330965638160706, "generator_grad_norm": 1.7841054201126099, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76923179626465, "peak_memory_bytes": 22206981120}
+{"step": 42, "generator_loss": 0.1833973079919815, "generator_grad_norm": 1.569561243057251, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96152210235596, "peak_memory_bytes": 22206981120}
+{"step": 43, "generator_loss": 0.07884911447763443, "generator_grad_norm": 2.113374948501587, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.91257834434509, "peak_memory_bytes": 22206981120}
+{"step": 44, "generator_loss": 0.23759742081165314, "generator_grad_norm": 1.6449737548828125, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.75697922706604, "peak_memory_bytes": 22206981120}
+{"step": 45, "generator_loss": 0.16423596441745758, "generator_grad_norm": 1.5028722286224365, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.881197452545166, "peak_memory_bytes": 22206981120}
+{"step": 46, "generator_loss": 0.11207359284162521, "generator_grad_norm": 1.990920901298523, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.74021244049072, "peak_memory_bytes": 22206981120}
+{"step": 47, "generator_loss": 0.14900077879428864, "generator_grad_norm": 1.817151427268982, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.89031672477722, "peak_memory_bytes": 22207118336}
+{"step": 48, "generator_loss": 0.1929827183485031, "generator_grad_norm": 1.2109702825546265, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.75825119018555, "peak_memory_bytes": 22207118336}
+{"step": 49, "generator_loss": 0.18891580402851105, "generator_grad_norm": 1.6280748844146729, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.894901514053345, "peak_memory_bytes": 22207118336}
+{"step": 50, "generator_loss": 0.1976214200258255, "generator_grad_norm": 1.7904722690582275, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.862993478775024, "peak_memory_bytes": 22207118336}
+{"step": 51, "generator_loss": 0.10816649347543716, "generator_grad_norm": 1.496626615524292, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76144862174988, "peak_memory_bytes": 22207118336}
+{"step": 52, "generator_loss": 0.15071019530296326, "generator_grad_norm": 2.2496583461761475, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.92556834220886, "peak_memory_bytes": 22207118336}
+{"step": 53, "generator_loss": 0.1976582258939743, "generator_grad_norm": 1.7934541702270508, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.92225456237793, "peak_memory_bytes": 22207118336}
+{"step": 54, "generator_loss": 0.08554039150476456, "generator_grad_norm": 2.1835532188415527, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76492786407471, "peak_memory_bytes": 22207118336}
+{"step": 55, "generator_loss": 0.17895278334617615, "generator_grad_norm": 1.6719539165496826, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86199188232422, "peak_memory_bytes": 22207118336}
+{"step": 56, "generator_loss": 0.2980744242668152, "generator_grad_norm": 1.8338161706924438, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.71792411804199, "peak_memory_bytes": 22207118336}
+{"step": 57, "generator_loss": 0.09514660388231277, "generator_grad_norm": 1.9055367708206177, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.855663776397705, "peak_memory_bytes": 22207118336}
+{"step": 58, "generator_loss": 0.13721737265586853, "generator_grad_norm": 1.4409229755401611, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.70604395866394, "peak_memory_bytes": 22207118336}
+{"step": 59, "generator_loss": 0.18440064787864685, "generator_grad_norm": 1.3634089231491089, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.81705856323242, "peak_memory_bytes": 22207118336}
+{"step": 60, "generator_loss": 0.39827412366867065, "generator_grad_norm": 1.9354075193405151, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.84340286254883, "peak_memory_bytes": 22207118336}
+{"step": 61, "generator_loss": 0.14426253736019135, "generator_grad_norm": 1.8744813203811646, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.782939434051514, "peak_memory_bytes": 22207118336}
+{"step": 62, "generator_loss": 0.4315720498561859, "generator_grad_norm": 1.861358880996704, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.87258696556091, "peak_memory_bytes": 22207118336}
+{"step": 63, "generator_loss": 0.2252703160047531, "generator_grad_norm": 1.1835598945617676, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.84951591491699, "peak_memory_bytes": 22207118336}
+{"step": 64, "generator_loss": 0.19086581468582153, "generator_grad_norm": 1.791168451309204, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.70482301712036, "peak_memory_bytes": 22207118336}
+{"step": 65, "generator_loss": 0.06410085409879684, "generator_grad_norm": 1.517551064491272, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.825825929641724, "peak_memory_bytes": 22207118336}
+{"step": 66, "generator_loss": 0.08699367195367813, "generator_grad_norm": 1.445365309715271, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.67346262931824, "peak_memory_bytes": 22207118336}
+{"step": 67, "generator_loss": 0.0781392753124237, "generator_grad_norm": 1.6335198879241943, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.79273200035095, "peak_memory_bytes": 22207118336}
+{"step": 68, "generator_loss": 0.3947698473930359, "generator_grad_norm": 1.6978559494018555, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66453313827515, "peak_memory_bytes": 22207118336}
+{"step": 69, "generator_loss": 0.21365517377853394, "generator_grad_norm": 2.271358013153076, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.79025650024414, "peak_memory_bytes": 22207118336}
+{"step": 70, "generator_loss": 0.12494263797998428, "generator_grad_norm": 1.65260648727417, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80284357070923, "peak_memory_bytes": 22207118336}
+{"step": 71, "generator_loss": 0.12745366990566254, "generator_grad_norm": 1.9492439031600952, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.704575538635254, "peak_memory_bytes": 22207118336}
+{"step": 72, "generator_loss": 0.14017222821712494, "generator_grad_norm": 1.0913259983062744, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.865525245666504, "peak_memory_bytes": 22207118336}
+{"step": 73, "generator_loss": 0.4774479866027832, "generator_grad_norm": 1.8631647825241089, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.827256202697754, "peak_memory_bytes": 22207118336}
+{"step": 74, "generator_loss": 0.32345542311668396, "generator_grad_norm": 1.5261541604995728, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.65160393714905, "peak_memory_bytes": 22207118336}
+{"step": 75, "generator_loss": 0.32232892513275146, "generator_grad_norm": 1.57761549949646, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.835142612457275, "peak_memory_bytes": 22207118336}
+{"step": 76, "generator_loss": 0.10670802742242813, "generator_grad_norm": 1.2530498504638672, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.663676500320435, "peak_memory_bytes": 22207118336}
+{"step": 77, "generator_loss": 0.2021353393793106, "generator_grad_norm": 1.7750661373138428, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.84876322746277, "peak_memory_bytes": 22207118336}
+{"step": 78, "generator_loss": 0.2725768983364105, "generator_grad_norm": 1.301364541053772, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.67252731323242, "peak_memory_bytes": 22207118336}
+{"step": 79, "generator_loss": 0.16312356293201447, "generator_grad_norm": 1.496948003768921, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.7915358543396, "peak_memory_bytes": 22207118336}
+{"step": 80, "generator_loss": 0.24257010221481323, "generator_grad_norm": 1.7060368061065674, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.8589186668396, "peak_memory_bytes": 22207118336}
+{"step": 81, "generator_loss": 0.11316300928592682, "generator_grad_norm": 0.7517905235290527, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.705838203430176, "peak_memory_bytes": 22207118336}
+{"step": 82, "generator_loss": 0.19950388371944427, "generator_grad_norm": 1.1980787515640259, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85356903076172, "peak_memory_bytes": 22207118336}
+{"step": 83, "generator_loss": 0.10085529834032059, "generator_grad_norm": 1.14894700050354, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85069751739502, "peak_memory_bytes": 22207118336}
+{"step": 84, "generator_loss": 0.4765048325061798, "generator_grad_norm": 1.5539790391921997, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66763496398926, "peak_memory_bytes": 22207118336}
+{"step": 85, "generator_loss": 0.09645943343639374, "generator_grad_norm": 1.5117207765579224, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.819966077804565, "peak_memory_bytes": 22207118336}
+{"step": 86, "generator_loss": 0.04516522213816643, "generator_grad_norm": 1.1207022666931152, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.6426522731781, "peak_memory_bytes": 22207118336}
+{"step": 87, "generator_loss": 0.1546209752559662, "generator_grad_norm": 1.4605077505111694, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80625557899475, "peak_memory_bytes": 22207118336}
+{"step": 88, "generator_loss": 0.11338809877634048, "generator_grad_norm": 1.4040987491607666, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66527462005615, "peak_memory_bytes": 22207118336}
+{"step": 89, "generator_loss": 0.4342181086540222, "generator_grad_norm": 1.2287545204162598, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.772387742996216, "peak_memory_bytes": 22207118336}
+{"step": 90, "generator_loss": 0.26641541719436646, "generator_grad_norm": 1.4647990465164185, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76701331138611, "peak_memory_bytes": 22207118336}
+{"step": 91, "generator_loss": 0.11470887809991837, "generator_grad_norm": 1.2308169603347778, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.63545870780945, "peak_memory_bytes": 22207118336}
+{"step": 92, "generator_loss": 0.07105366885662079, "generator_grad_norm": 1.6173112392425537, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.811002254486084, "peak_memory_bytes": 22207118336}
+{"step": 93, "generator_loss": 0.07540740072727203, "generator_grad_norm": 0.6796842813491821, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.78603792190552, "peak_memory_bytes": 22207118336}
+{"step": 94, "generator_loss": 0.4029463529586792, "generator_grad_norm": 1.2325291633605957, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.65994906425476, "peak_memory_bytes": 22207118336}
+{"step": 95, "generator_loss": 0.053321193903684616, "generator_grad_norm": 1.1912261247634888, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80720114707947, "peak_memory_bytes": 22207118336}
+{"step": 96, "generator_loss": 0.44524964690208435, "generator_grad_norm": 1.1646605730056763, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.669270753860474, "peak_memory_bytes": 22207118336}
+{"step": 97, "generator_loss": 0.3676246404647827, "generator_grad_norm": 0.7945242524147034, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.794057846069336, "peak_memory_bytes": 22207118336}
+{"step": 98, "generator_loss": 0.1692187488079071, "generator_grad_norm": 1.8839051723480225, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.636770486831665, "peak_memory_bytes": 22207118336}
+{"step": 99, "generator_loss": 0.10734278708696365, "generator_grad_norm": 1.7294666767120361, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.74717330932617, "peak_memory_bytes": 22207118336}
+{"step": 100, "generator_loss": 0.1598043441772461, "generator_grad_norm": 1.121717929840088, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.77630281448364, "peak_memory_bytes": 22207118336}
+{"step": 101, "generator_loss": 0.1984109729528427, "generator_grad_norm": 1.574268102645874, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.143136739730835, "peak_memory_bytes": 22207118336}
+{"step": 102, "generator_loss": 0.11785068362951279, "generator_grad_norm": 0.7268500328063965, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76971077919006, "peak_memory_bytes": 22207118336}
+{"step": 103, "generator_loss": 0.10634899139404297, "generator_grad_norm": 0.6620595455169678, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.65498232841492, "peak_memory_bytes": 22207118336}
+{"step": 104, "generator_loss": 0.4098441004753113, "generator_grad_norm": 1.2350507974624634, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.774662494659424, "peak_memory_bytes": 22207118336}
+{"step": 105, "generator_loss": 0.05807800218462944, "generator_grad_norm": 0.7025431990623474, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.62861132621765, "peak_memory_bytes": 22207118336}
+{"step": 106, "generator_loss": 0.06164713203907013, "generator_grad_norm": 1.8682787418365479, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.79497027397156, "peak_memory_bytes": 22207118336}
+{"step": 107, "generator_loss": 0.08294948190450668, "generator_grad_norm": 0.856795608997345, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.807666301727295, "peak_memory_bytes": 22207118336}
+{"step": 108, "generator_loss": 0.1857418566942215, "generator_grad_norm": 1.3859394788742065, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66919660568237, "peak_memory_bytes": 22207118336}
+{"step": 109, "generator_loss": 0.523881733417511, "generator_grad_norm": 1.2924517393112183, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.74103045463562, "peak_memory_bytes": 22207118336}
+{"step": 110, "generator_loss": 0.07563872635364532, "generator_grad_norm": 1.2275351285934448, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.64898085594177, "peak_memory_bytes": 22207118336}
+{"step": 111, "generator_loss": 0.23264054954051971, "generator_grad_norm": 0.8982031345367432, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.761643409729004, "peak_memory_bytes": 22207118336}
+{"step": 112, "generator_loss": 0.27945011854171753, "generator_grad_norm": 1.3006006479263306, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.78931450843811, "peak_memory_bytes": 22207118336}
+{"step": 113, "generator_loss": 0.2772520184516907, "generator_grad_norm": 0.9812349677085876, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.6306037902832, "peak_memory_bytes": 22207118336}
+{"step": 114, "generator_loss": 0.11895660310983658, "generator_grad_norm": 1.0431596040725708, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.727577209472656, "peak_memory_bytes": 22207118336}
+{"step": 115, "generator_loss": 0.10890975594520569, "generator_grad_norm": 1.0968310832977295, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.64460563659668, "peak_memory_bytes": 22207118336}
+{"step": 116, "generator_loss": 0.08057565987110138, "generator_grad_norm": 0.7288354635238647, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.767072916030884, "peak_memory_bytes": 22207118336}
+{"step": 117, "generator_loss": 0.28552141785621643, "generator_grad_norm": 0.7222070097923279, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.645081758499146, "peak_memory_bytes": 22207118336}
+{"step": 118, "generator_loss": 0.05057976394891739, "generator_grad_norm": 1.090677261352539, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80406403541565, "peak_memory_bytes": 22207118336}
+{"step": 119, "generator_loss": 0.1369611918926239, "generator_grad_norm": 0.5915815234184265, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.811376333236694, "peak_memory_bytes": 22207118336}
+{"step": 120, "generator_loss": 0.22802084684371948, "generator_grad_norm": 0.8891777992248535, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66960859298706, "peak_memory_bytes": 22207118336}
+{"step": 121, "generator_loss": 0.11770720779895782, "generator_grad_norm": 0.7738595604896545, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.83649015426636, "peak_memory_bytes": 22207118336}
+{"step": 122, "generator_loss": 0.07326442748308182, "generator_grad_norm": 0.8714878559112549, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.6821448802948, "peak_memory_bytes": 22207118336}
+{"step": 123, "generator_loss": 0.1593952625989914, "generator_grad_norm": 0.6362482905387878, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.90776062011719, "peak_memory_bytes": 22207118336}
+{"step": 124, "generator_loss": 0.08880941569805145, "generator_grad_norm": 0.7550555467605591, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.69906449317932, "peak_memory_bytes": 22207118336}
+{"step": 125, "generator_loss": 0.22726306319236755, "generator_grad_norm": 0.8486024141311646, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.877851247787476, "peak_memory_bytes": 22207118336}
+{"step": 126, "generator_loss": 0.33075159788131714, "generator_grad_norm": 0.7071317434310913, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.822322845458984, "peak_memory_bytes": 22207118336}
+{"step": 127, "generator_loss": 0.15914635360240936, "generator_grad_norm": 0.3759337067604065, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.818029165267944, "peak_memory_bytes": 22207118336}
+{"step": 128, "generator_loss": 0.10291738063097, "generator_grad_norm": 0.5276326537132263, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.8942174911499, "peak_memory_bytes": 22207118336}
+{"step": 129, "generator_loss": 0.21067799627780914, "generator_grad_norm": 0.9464908838272095, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.75065636634827, "peak_memory_bytes": 22207118336}
+{"step": 130, "generator_loss": 0.11703181266784668, "generator_grad_norm": 0.6535447239875793, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.91620326042175, "peak_memory_bytes": 22207118336}
+{"step": 131, "generator_loss": 0.25263696908950806, "generator_grad_norm": 0.7000232338905334, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.74502491950989, "peak_memory_bytes": 22207118336}
+{"step": 132, "generator_loss": 0.13378897309303284, "generator_grad_norm": 0.58815997838974, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.87606406211853, "peak_memory_bytes": 22207118336}
+{"step": 133, "generator_loss": 0.26069170236587524, "generator_grad_norm": 0.8928821682929993, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80268168449402, "peak_memory_bytes": 22207118336}
+{"step": 134, "generator_loss": 0.21071282029151917, "generator_grad_norm": 0.9145609736442566, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.926913022994995, "peak_memory_bytes": 22207118336}
+{"step": 135, "generator_loss": 0.06307131052017212, "generator_grad_norm": 0.5621073842048645, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98429870605469, "peak_memory_bytes": 22207118336}
+{"step": 136, "generator_loss": 0.13477405905723572, "generator_grad_norm": 0.6856816411018372, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.82456994056702, "peak_memory_bytes": 22207118336}
+{"step": 137, "generator_loss": 0.20405443012714386, "generator_grad_norm": 1.1022902727127075, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95421123504639, "peak_memory_bytes": 22207118336}
+{"step": 138, "generator_loss": 0.22960327565670013, "generator_grad_norm": 0.4853450655937195, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86044239997864, "peak_memory_bytes": 22207118336}
+{"step": 139, "generator_loss": 0.09174457937479019, "generator_grad_norm": 0.4998477101325989, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98919367790222, "peak_memory_bytes": 22207118336}
+{"step": 140, "generator_loss": 0.05428297445178032, "generator_grad_norm": 0.6890667080879211, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97464466094971, "peak_memory_bytes": 22207118336}
+{"step": 141, "generator_loss": 0.13453836739063263, "generator_grad_norm": 0.7356487512588501, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.87828040122986, "peak_memory_bytes": 22207118336}
+{"step": 142, "generator_loss": 0.33643215894699097, "generator_grad_norm": 0.4792184829711914, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.988914251327515, "peak_memory_bytes": 22207118336}
+{"step": 143, "generator_loss": 0.054217249155044556, "generator_grad_norm": 0.6441846489906311, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.835625648498535, "peak_memory_bytes": 22207118336}
+{"step": 144, "generator_loss": 0.04892465099692345, "generator_grad_norm": 1.20295250415802, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.916372776031494, "peak_memory_bytes": 22207118336}
+{"step": 145, "generator_loss": 0.13452447950839996, "generator_grad_norm": 0.5148499608039856, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.812121629714966, "peak_memory_bytes": 22207118336}
+{"step": 146, "generator_loss": 0.040658410638570786, "generator_grad_norm": 1.0761929750442505, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94335865974426, "peak_memory_bytes": 22207118336}
+{"step": 147, "generator_loss": 0.1905137002468109, "generator_grad_norm": 0.38416582345962524, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.896130323410034, "peak_memory_bytes": 22207118336}
+{"step": 148, "generator_loss": 0.10900343209505081, "generator_grad_norm": 0.4615156352519989, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80624175071716, "peak_memory_bytes": 22207118336}
+{"step": 149, "generator_loss": 0.09254646301269531, "generator_grad_norm": 0.5535904765129089, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98242163658142, "peak_memory_bytes": 22207118336}
+{"step": 150, "generator_loss": 0.14974164962768555, "generator_grad_norm": 0.593259871006012, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.73344445228577, "peak_memory_bytes": 22207118336}
+{"step": 151, "generator_loss": 0.06853128969669342, "generator_grad_norm": 0.46588459610939026, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.917866230010986, "peak_memory_bytes": 22207118336}
+{"step": 152, "generator_loss": 0.3614100217819214, "generator_grad_norm": 0.5073930621147156, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.74210453033447, "peak_memory_bytes": 22207118336}
+{"step": 153, "generator_loss": 0.07388052344322205, "generator_grad_norm": 0.2602158188819885, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.8895218372345, "peak_memory_bytes": 22207118336}
+{"step": 154, "generator_loss": 0.044851697981357574, "generator_grad_norm": 0.42720678448677063, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.884862661361694, "peak_memory_bytes": 22207118336}
+{"step": 155, "generator_loss": 0.07534617930650711, "generator_grad_norm": 0.755936324596405, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.719661235809326, "peak_memory_bytes": 22207118336}
+{"step": 156, "generator_loss": 0.09686053544282913, "generator_grad_norm": 0.7743855118751526, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85826897621155, "peak_memory_bytes": 22207118336}
+{"step": 157, "generator_loss": 0.09431374818086624, "generator_grad_norm": 0.5788172483444214, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.72025656700134, "peak_memory_bytes": 22207118336}
+{"step": 158, "generator_loss": 0.09127192944288254, "generator_grad_norm": 0.4649401009082794, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.848926305770874, "peak_memory_bytes": 22207118336}
+{"step": 159, "generator_loss": 0.08054102212190628, "generator_grad_norm": 0.33810973167419434, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.71176743507385, "peak_memory_bytes": 22207118336}
+{"step": 160, "generator_loss": 0.052245959639549255, "generator_grad_norm": 0.3660430610179901, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85009503364563, "peak_memory_bytes": 22207118336}
+{"step": 161, "generator_loss": 0.09968896210193634, "generator_grad_norm": 0.6541793942451477, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.89983582496643, "peak_memory_bytes": 22207118336}
+{"step": 162, "generator_loss": 0.1392759382724762, "generator_grad_norm": 0.33457422256469727, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.77232265472412, "peak_memory_bytes": 22207118336}
+{"step": 163, "generator_loss": 0.08443567901849747, "generator_grad_norm": 0.2789621651172638, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.92086172103882, "peak_memory_bytes": 22207118336}
+{"step": 164, "generator_loss": 0.07604159414768219, "generator_grad_norm": 0.3477323353290558, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.766090631484985, "peak_memory_bytes": 22207118336}
+{"step": 165, "generator_loss": 0.09156201034784317, "generator_grad_norm": 0.475818932056427, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.79659676551819, "peak_memory_bytes": 22207118336}
+{"step": 166, "generator_loss": 0.06923417001962662, "generator_grad_norm": 0.6561019420623779, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.72503638267517, "peak_memory_bytes": 22207118336}
+{"step": 167, "generator_loss": 0.13126005232334137, "generator_grad_norm": 0.5822004675865173, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.84088683128357, "peak_memory_bytes": 22207118336}
+{"step": 168, "generator_loss": 0.032114703208208084, "generator_grad_norm": 0.49446505308151245, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.87023043632507, "peak_memory_bytes": 22207118336}
+{"step": 169, "generator_loss": 0.18781651556491852, "generator_grad_norm": 0.5328198671340942, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.68285584449768, "peak_memory_bytes": 22207118336}
+{"step": 170, "generator_loss": 0.1160624623298645, "generator_grad_norm": 0.4395141005516052, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.78830099105835, "peak_memory_bytes": 22207118336}
+{"step": 171, "generator_loss": 0.14174671471118927, "generator_grad_norm": 0.30042076110839844, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.712201833724976, "peak_memory_bytes": 22207118336}
+{"step": 172, "generator_loss": 0.05593271180987358, "generator_grad_norm": 0.4837401509284973, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80807280540466, "peak_memory_bytes": 22207118336}
+{"step": 173, "generator_loss": 0.26347318291664124, "generator_grad_norm": 0.43765294551849365, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.69795227050781, "peak_memory_bytes": 22207118336}
+{"step": 174, "generator_loss": 0.08855048567056656, "generator_grad_norm": 0.3454621136188507, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.80228853225708, "peak_memory_bytes": 22207118336}
+{"step": 175, "generator_loss": 0.17608048021793365, "generator_grad_norm": 0.6615318059921265, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.77237010002136, "peak_memory_bytes": 22207118336}
+{"step": 176, "generator_loss": 0.09613776206970215, "generator_grad_norm": 0.582581102848053, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.68240523338318, "peak_memory_bytes": 22207118336}
+{"step": 177, "generator_loss": 0.18174821138381958, "generator_grad_norm": 0.48736128211021423, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.797709703445435, "peak_memory_bytes": 22207118336}
+{"step": 178, "generator_loss": 0.38082069158554077, "generator_grad_norm": 0.681738555431366, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.67607498168945, "peak_memory_bytes": 22207118336}
+{"step": 179, "generator_loss": 0.0786067470908165, "generator_grad_norm": 0.3669721186161041, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.81005072593689, "peak_memory_bytes": 22207118336}
+{"step": 180, "generator_loss": 0.0856500118970871, "generator_grad_norm": 0.5466479659080505, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76675605773926, "peak_memory_bytes": 22207118336}
+{"step": 181, "generator_loss": 0.058225661516189575, "generator_grad_norm": 0.4041939377784729, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.660924673080444, "peak_memory_bytes": 22207118336}
+{"step": 182, "generator_loss": 0.2636546492576599, "generator_grad_norm": 0.6231318712234497, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.83431553840637, "peak_memory_bytes": 22207118336}
+{"step": 183, "generator_loss": 0.0704910010099411, "generator_grad_norm": 0.22858759760856628, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.656391620635986, "peak_memory_bytes": 22207118336}
+{"step": 184, "generator_loss": 0.11076851189136505, "generator_grad_norm": 0.5283437967300415, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.76090860366821, "peak_memory_bytes": 22207118336}
+{"step": 185, "generator_loss": 0.1294175535440445, "generator_grad_norm": 0.5911198854446411, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.6395263671875, "peak_memory_bytes": 22207118336}
+{"step": 186, "generator_loss": 0.11945979297161102, "generator_grad_norm": 0.44618454575538635, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.82966494560242, "peak_memory_bytes": 22207118336}
+{"step": 187, "generator_loss": 0.11358052492141724, "generator_grad_norm": 0.33687111735343933, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85239887237549, "peak_memory_bytes": 22207118336}
+{"step": 188, "generator_loss": 0.1364806592464447, "generator_grad_norm": 0.3996356725692749, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.69481420516968, "peak_memory_bytes": 22207118336}
+{"step": 189, "generator_loss": 0.13921917974948883, "generator_grad_norm": 0.6863322257995605, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.81850481033325, "peak_memory_bytes": 22207118336}
+{"step": 190, "generator_loss": 0.17315605282783508, "generator_grad_norm": 0.6243394017219543, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.697619915008545, "peak_memory_bytes": 22207118336}
+{"step": 191, "generator_loss": 0.05595165491104126, "generator_grad_norm": 0.307320237159729, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.85112380981445, "peak_memory_bytes": 22207118336}
+{"step": 192, "generator_loss": 0.043159253895282745, "generator_grad_norm": 0.3174380362033844, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.83359456062317, "peak_memory_bytes": 22207118336}
+{"step": 193, "generator_loss": 0.1979258954524994, "generator_grad_norm": 0.6831952929496765, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.69136023521423, "peak_memory_bytes": 22207118336}
+{"step": 194, "generator_loss": 0.19321943819522858, "generator_grad_norm": 0.549832820892334, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.78218984603882, "peak_memory_bytes": 22207118336}
+{"step": 195, "generator_loss": 0.09127001464366913, "generator_grad_norm": 0.4105549156665802, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.641507148742676, "peak_memory_bytes": 22207118336}
+{"step": 196, "generator_loss": 0.08986713737249374, "generator_grad_norm": 0.3296584188938141, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86053490638733, "peak_memory_bytes": 22207118336}
+{"step": 197, "generator_loss": 0.19208924472332, "generator_grad_norm": 0.4328521490097046, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.661096811294556, "peak_memory_bytes": 22207118336}
+{"step": 198, "generator_loss": 0.29705455899238586, "generator_grad_norm": 0.753167986869812, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.90293049812317, "peak_memory_bytes": 22207118336}
+{"step": 199, "generator_loss": 0.055089958012104034, "generator_grad_norm": 0.48070377111434937, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.86617588996887, "peak_memory_bytes": 22207118336}
+{"step": 200, "generator_loss": 0.06518029421567917, "generator_grad_norm": 0.4703660309314728, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.66055488586426, "peak_memory_bytes": 22207118336}

outputs/temporal_nope/a4_tdrope_hard_lr2e7_probe200_seed20260805_kerrigan/validation_metrics.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "validation_loss": 0.2635482151526958, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 22.61892604827881, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 100, "validation_loss": 0.14690994436386973, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.253740072250366, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 200, "validation_loss": 0.1128241450060159, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.328860998153687, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}

outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/checkpoint_manifest.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "type": "immutable_base_checkpoint_reference", "path": "/data/fengjiaqi/causal_forcing/checkpoints/framewise/causal_cd.pt", "checkpoint_key": "generator_ema", "sha256": "c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22", "optimizer_state": "reset"}
+{"step": 100, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000100/model.pt", "sha256": "0f0001d419f5e22e14017e98b581e1e54d7fc6f1327d4745106dde9fa4444880", "size_bytes": 5676282858, "contains": ["generator"]}
+{"step": 200, "path": "/home/jiaqi/NoPE/outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/checkpoint_model_000200/model.pt", "sha256": "1e85b4ece877629d20ada99adcf0974700a1449f0af76c311b08d74260471d51", "size_bytes": 5676282858, "contains": ["generator"]}

outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/resolved_config.yaml

@@ -0,0 +1,123 @@
+experiment:
+  id: a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan
+  seed: 20260805
+  source_revision: bb190459d99b074b5803ed6ba0b5091b16d5585d
+  output_root: outputs/temporal_nope
+  report_root: reports/temporal_nope
+  tier: A
+  method: Annealed Temporal DroPE
+generator_ckpt: checkpoints/framewise/causal_cd.pt
+generator_ckpt_sha256: c951a6b4804cd637fecfc857e9a59d54b6e4cd7846c38360baa3cf3b525f0d22
+checkpoint_key: generator_ema
+strict_checkpoint_load: true
+model_name: Wan2.1-T2V-1.3B
+generator_task: causal_video
+causal: true
+mixed_precision: true
+gradient_checkpointing: true
+num_frame_per_block: 1
+independent_first_frame: false
+num_training_frames: 21
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+height: 480
+width: 832
+num_frames: 81
+fps: 16
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+warp_denoising_step: true
+num_train_timestep: 1000
+timestep_shift: 5.0
+guidance_scale: 3.0
+negative_prompt: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+prompt_embedding_cache_path: /data/fengjiaqi/Causal_forcing/prompt_cache/chunkwise_umt5_bf16_lmdb
+rank0_preload_generator_ckpt: true
+model_kwargs:
+  timestep_shift: 5.0
+position_encoding:
+  spatial_mode: rope
+  temporal_mode: rope
+  temporal_rope_alpha: 0.0
+  temporal_rope_schedule: cosine
+  temporal_rope_anneal_start: 0
+  temporal_rope_anneal_end: 40
+  temporal_layer_pattern: all
+  temporal_position_scale: 1.0
+  temporal_rope_theta: 10000.0
+  riflex_k: null
+  riflex_target_length: null
+  riflex_observed_repetition_length: null
+attention_scaling:
+  enabled: false
+  mode: none
+  coefficient: 0.0
+  train_length: 21
+  length_ratio: null
+  per_head_scales: null
+analysis:
+  record_attention_stats: false
+  sampled_layers:
+  - 0
+  - 0.5
+  - -1
+  sampled_spatial_queries: 64
+  target_visible_lengths:
+  - 21
+  - 42
+  - 84
+  train_length: 21
+training:
+  enabled: true
+  objective: teacher_forced_flow_matching
+  data_path: dataset/clean_data
+  split_manifest: artifacts/data_splits/cpt_train_val_seed20260805.json
+  fixed_latent_length: 21
+  batch_size_per_gpu: 1
+  total_batch_size: 8
+  max_steps: 200
+  checkpoint_steps:
+  - 0
+  - 100
+  - 200
+  optimizer: adamw
+  optimizer_reset: true
+  lr: 1.0e-07
+  original_pretrain_peak_lr: 2.0e-06
+  betas:
+  - 0.0
+  - 0.999
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  bf16: true
+  ema_enabled: false
+  num_workers: 8
+  log_every: 1
+  validation_every: 100
+  validation_batches_per_rank: 4
+  validation_seed: 20260805
+evaluation:
+  length_ratios:
+  - 1.0
+  - 1.5
+  - 2.0
+  - 3.0
+  - 4.0
+  validation_prompt_file: eval/prompts_temporal_nope_val.json
+  test_prompt_file: eval/prompts_temporal_nope_test.json
+  validation_seeds:
+  - 0
+  - 1
+  test_seeds:
+  - 0
+  - 1
+  - 2
+  - 3
+  paired_longest_rollout: true

outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/train.log

@@ -0,0 +1,56 @@
+W0806 17:35:02.258000 2931479 site-packages/torch/distributed/run.py:793] 
+W0806 17:35:02.258000 2931479 site-packages/torch/distributed/run.py:793] *****************************************
+W0806 17:35:02.258000 2931479 site-packages/torch/distributed/run.py:793] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0806 17:35:02.258000 2931479 site-packages/torch/distributed/run.py:793] *****************************************
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+CAUSAL_DISABLE_FLEX_ATTENTION=1 -> using segmented FlashAttention fallback
+[rank4]:[W806 17:35:11.398747864 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank3]:[W806 17:35:12.973127437 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank6]:[W806 17:35:12.076794181 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank1]:[W806 17:35:12.397289901 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+Rank 0 preloading generator from checkpoints/framewise/causal_cd.pt
+[rank2]:[W806 17:35:12.422225633 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank5]:[W806 17:35:12.423958046 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank7]:[W806 17:35:12.427401743 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+[rank0]:[W806 17:35:13.673997724 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
+DATASET TRAIN 6249 VALIDATION 256 SPLIT /home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json
+validation step=0 loss=0.06264185 alpha=1.000000 samples=32 seconds=23.283
+step=1 loss=0.16898617 grad_norm=0.284414 alpha=1.000000 seconds=34.007
+step=2 loss=0.08545000 grad_norm=0.263569 alpha=0.998459 seconds=34.050
+W0806 17:36:50.927000 2931479 site-packages/torch/distributed/elastic/agent/server/api.py:704] Received Signals.SIGTERM death signal, shutting down workers
+W0806 17:36:50.929000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931561 closing signal SIGTERM
+W0806 17:36:50.930000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931562 closing signal SIGTERM
+W0806 17:36:50.932000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931563 closing signal SIGTERM
+W0806 17:36:50.933000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931564 closing signal SIGTERM
+W0806 17:36:50.933000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931565 closing signal SIGTERM
+W0806 17:36:50.933000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931566 closing signal SIGTERM
+W0806 17:36:50.934000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931567 closing signal SIGTERM
+W0806 17:36:50.934000 2931479 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 2931568 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
+    return f(*args, **kwargs)
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 919, in main
+    run(args)
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 910, in run
+    elastic_launch(
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent
+    result = agent.run()
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper
+    result = f(*args, **kwargs)
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 696, in run
+    result = self._invoke_run(role)
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 855, in _invoke_run
+    time.sleep(monitor_interval)
+  File "/home/jiaqi/miniconda3/envs/causal_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 2931479 got signal: 15

outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/training_metrics.jsonl

@@ -0,0 +1,200 @@
+{"step": 1, "generator_loss": 0.16898617148399353, "generator_grad_norm": 0.284491628408432, "temporal_rope_alpha": 1.0, "seconds_per_step": 33.778828859329224, "peak_memory_bytes": 20785906176}
+{"step": 2, "generator_loss": 0.0854455977678299, "generator_grad_norm": 0.263572096824646, "temporal_rope_alpha": 0.998458666866564, "seconds_per_step": 33.768771171569824, "peak_memory_bytes": 22206258176}
+{"step": 3, "generator_loss": 0.07995328307151794, "generator_grad_norm": 0.31424424052238464, "temporal_rope_alpha": 0.9938441702975689, "seconds_per_step": 34.08175706863403, "peak_memory_bytes": 22206258176}
+{"step": 4, "generator_loss": 0.11514420807361603, "generator_grad_norm": 0.2933346629142761, "temporal_rope_alpha": 0.9861849601988383, "seconds_per_step": 33.98695087432861, "peak_memory_bytes": 22206258176}
+{"step": 5, "generator_loss": 0.040942445397377014, "generator_grad_norm": 0.4405408203601837, "temporal_rope_alpha": 0.9755282581475768, "seconds_per_step": 34.21472120285034, "peak_memory_bytes": 22206694400}
+{"step": 6, "generator_loss": 0.050162408500909805, "generator_grad_norm": 0.24900391697883606, "temporal_rope_alpha": 0.9619397662556434, "seconds_per_step": 33.99621295928955, "peak_memory_bytes": 22206694400}
+{"step": 7, "generator_loss": 0.12222461402416229, "generator_grad_norm": 0.3024124801158905, "temporal_rope_alpha": 0.9455032620941839, "seconds_per_step": 34.12749767303467, "peak_memory_bytes": 22206694400}
+{"step": 8, "generator_loss": 0.04225009307265282, "generator_grad_norm": 0.2847746014595032, "temporal_rope_alpha": 0.9263200821770461, "seconds_per_step": 34.09383273124695, "peak_memory_bytes": 22206694400}
+{"step": 9, "generator_loss": 0.04081284999847412, "generator_grad_norm": 0.3021336793899536, "temporal_rope_alpha": 0.9045084971874737, "seconds_per_step": 33.960413455963135, "peak_memory_bytes": 22206694400}
+{"step": 10, "generator_loss": 0.12034216523170471, "generator_grad_norm": 0.3669663667678833, "temporal_rope_alpha": 0.8802029828000155, "seconds_per_step": 34.12534761428833, "peak_memory_bytes": 22206694400}
+{"step": 11, "generator_loss": 0.05508280545473099, "generator_grad_norm": 0.28416967391967773, "temporal_rope_alpha": 0.8535533905932737, "seconds_per_step": 34.051191329956055, "peak_memory_bytes": 22206694400}
+{"step": 12, "generator_loss": 0.041885945945978165, "generator_grad_norm": 0.2504422664642334, "temporal_rope_alpha": 0.8247240241650917, "seconds_per_step": 34.19885540008545, "peak_memory_bytes": 22206694400}
+{"step": 13, "generator_loss": 0.0756063386797905, "generator_grad_norm": 0.28763529658317566, "temporal_rope_alpha": 0.7938926261462366, "seconds_per_step": 34.18347120285034, "peak_memory_bytes": 22206694400}
+{"step": 14, "generator_loss": 0.12995368242263794, "generator_grad_norm": 0.31209397315979004, "temporal_rope_alpha": 0.7612492823579744, "seconds_per_step": 34.048121213912964, "peak_memory_bytes": 22206694400}
+{"step": 15, "generator_loss": 0.026575341820716858, "generator_grad_norm": 0.3550294041633606, "temporal_rope_alpha": 0.7269952498697734, "seconds_per_step": 34.14352798461914, "peak_memory_bytes": 22206694400}
+{"step": 16, "generator_loss": 0.0804247334599495, "generator_grad_norm": 0.3621918857097626, "temporal_rope_alpha": 0.6913417161825449, "seconds_per_step": 34.03455114364624, "peak_memory_bytes": 22206694400}
+{"step": 17, "generator_loss": 0.19613932073116302, "generator_grad_norm": 0.4811260402202606, "temporal_rope_alpha": 0.6545084971874737, "seconds_per_step": 34.15198588371277, "peak_memory_bytes": 22206694400}
+{"step": 18, "generator_loss": 0.08475203812122345, "generator_grad_norm": 0.5050399303436279, "temporal_rope_alpha": 0.6167226819279528, "seconds_per_step": 34.024232625961304, "peak_memory_bytes": 22206694400}
+{"step": 19, "generator_loss": 0.07170533388853073, "generator_grad_norm": 0.734170138835907, "temporal_rope_alpha": 0.5782172325201155, "seconds_per_step": 34.13564491271973, "peak_memory_bytes": 22206694400}
+{"step": 20, "generator_loss": 0.12927691638469696, "generator_grad_norm": 0.6722806692123413, "temporal_rope_alpha": 0.5392295478639225, "seconds_per_step": 34.15514636039734, "peak_memory_bytes": 22206831616}
+{"step": 21, "generator_loss": 0.06301255524158478, "generator_grad_norm": 1.11443293094635, "temporal_rope_alpha": 0.5, "seconds_per_step": 34.06606316566467, "peak_memory_bytes": 22206831616}
+{"step": 22, "generator_loss": 0.26007938385009766, "generator_grad_norm": 1.5868905782699585, "temporal_rope_alpha": 0.4607704521360776, "seconds_per_step": 34.166645526885986, "peak_memory_bytes": 22206831616}
+{"step": 23, "generator_loss": 0.15784652531147003, "generator_grad_norm": 1.8950388431549072, "temporal_rope_alpha": 0.42178276747988447, "seconds_per_step": 34.154855489730835, "peak_memory_bytes": 22206981120}
+{"step": 24, "generator_loss": 0.1104511022567749, "generator_grad_norm": 1.6543368101119995, "temporal_rope_alpha": 0.38327731807204746, "seconds_per_step": 34.005266189575195, "peak_memory_bytes": 22206981120}
+{"step": 25, "generator_loss": 0.05954372510313988, "generator_grad_norm": 2.096811532974243, "temporal_rope_alpha": 0.34549150281252633, "seconds_per_step": 34.16701555252075, "peak_memory_bytes": 22206981120}
+{"step": 26, "generator_loss": 0.12749837338924408, "generator_grad_norm": 1.985190510749817, "temporal_rope_alpha": 0.30865828381745514, "seconds_per_step": 34.00892663002014, "peak_memory_bytes": 22206981120}
+{"step": 27, "generator_loss": 0.19301968812942505, "generator_grad_norm": 1.663165807723999, "temporal_rope_alpha": 0.2730047501302266, "seconds_per_step": 34.13392162322998, "peak_memory_bytes": 22206981120}
+{"step": 28, "generator_loss": 0.07829982787370682, "generator_grad_norm": 1.6822773218154907, "temporal_rope_alpha": 0.2387507176420256, "seconds_per_step": 34.051541566848755, "peak_memory_bytes": 22206981120}
+{"step": 29, "generator_loss": 0.3043322265148163, "generator_grad_norm": 1.6721601486206055, "temporal_rope_alpha": 0.2061073738537635, "seconds_per_step": 34.13159918785095, "peak_memory_bytes": 22206981120}
+{"step": 30, "generator_loss": 0.14562925696372986, "generator_grad_norm": 2.439708948135376, "temporal_rope_alpha": 0.17527597583490823, "seconds_per_step": 34.160282373428345, "peak_memory_bytes": 22206981120}
+{"step": 31, "generator_loss": 0.44621238112449646, "generator_grad_norm": 2.6717348098754883, "temporal_rope_alpha": 0.14644660940672627, "seconds_per_step": 34.114821910858154, "peak_memory_bytes": 22206981120}
+{"step": 32, "generator_loss": 0.07767733931541443, "generator_grad_norm": 2.059067726135254, "temporal_rope_alpha": 0.11979701719998453, "seconds_per_step": 34.18035531044006, "peak_memory_bytes": 22206981120}
+{"step": 33, "generator_loss": 0.17118152976036072, "generator_grad_norm": 2.5404810905456543, "temporal_rope_alpha": 0.09549150281252633, "seconds_per_step": 34.232258796691895, "peak_memory_bytes": 22206981120}
+{"step": 34, "generator_loss": 0.4476866126060486, "generator_grad_norm": 2.1613705158233643, "temporal_rope_alpha": 0.07367991782295391, "seconds_per_step": 33.99373745918274, "peak_memory_bytes": 22206981120}
+{"step": 35, "generator_loss": 0.14872799813747406, "generator_grad_norm": 1.9571459293365479, "temporal_rope_alpha": 0.054496737905816106, "seconds_per_step": 34.129433393478394, "peak_memory_bytes": 22206981120}
+{"step": 36, "generator_loss": 0.2914104163646698, "generator_grad_norm": 2.2094767093658447, "temporal_rope_alpha": 0.03806023374435663, "seconds_per_step": 34.031336307525635, "peak_memory_bytes": 22206981120}
+{"step": 37, "generator_loss": 0.4429017901420593, "generator_grad_norm": 2.6182491779327393, "temporal_rope_alpha": 0.024471741852423234, "seconds_per_step": 34.169673919677734, "peak_memory_bytes": 22206981120}
+{"step": 38, "generator_loss": 0.2155512422323227, "generator_grad_norm": 2.184539318084717, "temporal_rope_alpha": 0.013815039801161721, "seconds_per_step": 34.044026136398315, "peak_memory_bytes": 22206981120}
+{"step": 39, "generator_loss": 0.191231369972229, "generator_grad_norm": 2.613896369934082, "temporal_rope_alpha": 0.00615582970243117, "seconds_per_step": 34.10932207107544, "peak_memory_bytes": 22206981120}
+{"step": 40, "generator_loss": 0.21448974311351776, "generator_grad_norm": 2.340555429458618, "temporal_rope_alpha": 0.001541333133436018, "seconds_per_step": 34.17858123779297, "peak_memory_bytes": 22206981120}
+{"step": 41, "generator_loss": 0.27764594554901123, "generator_grad_norm": 2.508953332901001, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02035355567932, "peak_memory_bytes": 22206981120}
+{"step": 42, "generator_loss": 0.22096046805381775, "generator_grad_norm": 2.226184606552124, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16349768638611, "peak_memory_bytes": 22206981120}
+{"step": 43, "generator_loss": 0.10185729712247849, "generator_grad_norm": 2.7237651348114014, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12702798843384, "peak_memory_bytes": 22206981120}
+{"step": 44, "generator_loss": 0.2868613004684448, "generator_grad_norm": 2.2940099239349365, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.040140867233276, "peak_memory_bytes": 22206981120}
+{"step": 45, "generator_loss": 0.19006837904453278, "generator_grad_norm": 2.173109769821167, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09494924545288, "peak_memory_bytes": 22206981120}
+{"step": 46, "generator_loss": 0.13861791789531708, "generator_grad_norm": 2.369474411010742, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97444272041321, "peak_memory_bytes": 22206981120}
+{"step": 47, "generator_loss": 0.18528422713279724, "generator_grad_norm": 2.4844961166381836, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.08288025856018, "peak_memory_bytes": 22207118336}
+{"step": 48, "generator_loss": 0.22300194203853607, "generator_grad_norm": 1.825470209121704, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.971373319625854, "peak_memory_bytes": 22207118336}
+{"step": 49, "generator_loss": 0.22882811725139618, "generator_grad_norm": 1.8829690217971802, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09381604194641, "peak_memory_bytes": 22207118336}
+{"step": 50, "generator_loss": 0.23621749877929688, "generator_grad_norm": 2.2569973468780518, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11077952384949, "peak_memory_bytes": 22207118336}
+{"step": 51, "generator_loss": 0.14101925492286682, "generator_grad_norm": 1.9465354681015015, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00685119628906, "peak_memory_bytes": 22207118336}
+{"step": 52, "generator_loss": 0.20145121216773987, "generator_grad_norm": 2.4571776390075684, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.13969588279724, "peak_memory_bytes": 22207118336}
+{"step": 53, "generator_loss": 0.2352861762046814, "generator_grad_norm": 2.096961498260498, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14370393753052, "peak_memory_bytes": 22207118336}
+{"step": 54, "generator_loss": 0.10611822456121445, "generator_grad_norm": 2.5278193950653076, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96879553794861, "peak_memory_bytes": 22207118336}
+{"step": 55, "generator_loss": 0.20699046552181244, "generator_grad_norm": 1.908884048461914, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12487196922302, "peak_memory_bytes": 22207118336}
+{"step": 56, "generator_loss": 0.3511064052581787, "generator_grad_norm": 2.2663543224334717, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.973044872283936, "peak_memory_bytes": 22207118336}
+{"step": 57, "generator_loss": 0.116619773209095, "generator_grad_norm": 1.9825632572174072, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10968518257141, "peak_memory_bytes": 22207118336}
+{"step": 58, "generator_loss": 0.1550961434841156, "generator_grad_norm": 1.922825574874878, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97564125061035, "peak_memory_bytes": 22207118336}
+{"step": 59, "generator_loss": 0.2083306610584259, "generator_grad_norm": 1.8369580507278442, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.08889961242676, "peak_memory_bytes": 22207118336}
+{"step": 60, "generator_loss": 0.47199976444244385, "generator_grad_norm": 2.2533018589019775, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12647771835327, "peak_memory_bytes": 22207118336}
+{"step": 61, "generator_loss": 0.16972488164901733, "generator_grad_norm": 2.2946250438690186, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.04601812362671, "peak_memory_bytes": 22207118336}
+{"step": 62, "generator_loss": 0.5125097632408142, "generator_grad_norm": 2.229633331298828, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.126219749450684, "peak_memory_bytes": 22207118336}
+{"step": 63, "generator_loss": 0.2760005295276642, "generator_grad_norm": 1.771531581878662, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.140405893325806, "peak_memory_bytes": 22207118336}
+{"step": 64, "generator_loss": 0.21866196393966675, "generator_grad_norm": 2.084113359451294, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98496413230896, "peak_memory_bytes": 22207118336}
+{"step": 65, "generator_loss": 0.08137930184602737, "generator_grad_norm": 1.7600747346878052, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07961559295654, "peak_memory_bytes": 22207118336}
+{"step": 66, "generator_loss": 0.11140146851539612, "generator_grad_norm": 1.618826985359192, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95601487159729, "peak_memory_bytes": 22207118336}
+{"step": 67, "generator_loss": 0.10882661491632462, "generator_grad_norm": 2.130786895751953, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.06711149215698, "peak_memory_bytes": 22207118336}
+{"step": 68, "generator_loss": 0.4567353427410126, "generator_grad_norm": 1.932231068611145, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.96810555458069, "peak_memory_bytes": 22207118336}
+{"step": 69, "generator_loss": 0.2832255959510803, "generator_grad_norm": 2.4776179790496826, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09631419181824, "peak_memory_bytes": 22207118336}
+{"step": 70, "generator_loss": 0.1557394564151764, "generator_grad_norm": 1.8042532205581665, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07814645767212, "peak_memory_bytes": 22207118336}
+{"step": 71, "generator_loss": 0.17194466292858124, "generator_grad_norm": 2.0446934700012207, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9805691242218, "peak_memory_bytes": 22207118336}
+{"step": 72, "generator_loss": 0.1827496886253357, "generator_grad_norm": 1.4635121822357178, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.137293577194214, "peak_memory_bytes": 22207118336}
+{"step": 73, "generator_loss": 0.5746272206306458, "generator_grad_norm": 1.9792851209640503, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.150922775268555, "peak_memory_bytes": 22207118336}
+{"step": 74, "generator_loss": 0.41170793771743774, "generator_grad_norm": 1.7321598529815674, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.93162441253662, "peak_memory_bytes": 22207118336}
+{"step": 75, "generator_loss": 0.40445178747177124, "generator_grad_norm": 1.8297381401062012, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.072144508361816, "peak_memory_bytes": 22207118336}
+{"step": 76, "generator_loss": 0.14829649031162262, "generator_grad_norm": 1.613633632659912, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94431138038635, "peak_memory_bytes": 22207118336}
+{"step": 77, "generator_loss": 0.27055296301841736, "generator_grad_norm": 2.0182392597198486, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09240698814392, "peak_memory_bytes": 22207118336}
+{"step": 78, "generator_loss": 0.3680310547351837, "generator_grad_norm": 1.9426908493041992, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.957685232162476, "peak_memory_bytes": 22207118336}
+{"step": 79, "generator_loss": 0.21197496354579926, "generator_grad_norm": 1.9631237983703613, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.39937996864319, "peak_memory_bytes": 22207118336}
+{"step": 80, "generator_loss": 0.3143959939479828, "generator_grad_norm": 1.9137749671936035, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.22317457199097, "peak_memory_bytes": 22207118336}
+{"step": 81, "generator_loss": 0.12517160177230835, "generator_grad_norm": 1.1159050464630127, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98964047431946, "peak_memory_bytes": 22207118336}
+{"step": 82, "generator_loss": 0.2825927436351776, "generator_grad_norm": 1.9179617166519165, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17833638191223, "peak_memory_bytes": 22207118336}
+{"step": 83, "generator_loss": 0.12693153321743011, "generator_grad_norm": 1.521921992301941, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17185640335083, "peak_memory_bytes": 22207118336}
+{"step": 84, "generator_loss": 0.602593719959259, "generator_grad_norm": 1.8570040464401245, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.994930028915405, "peak_memory_bytes": 22207118336}
+{"step": 85, "generator_loss": 0.12741465866565704, "generator_grad_norm": 2.076425075531006, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.129106760025024, "peak_memory_bytes": 22207118336}
+{"step": 86, "generator_loss": 0.0538465715944767, "generator_grad_norm": 1.7526161670684814, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9447762966156, "peak_memory_bytes": 22207118336}
+{"step": 87, "generator_loss": 0.2039949744939804, "generator_grad_norm": 2.082354784011841, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09718203544617, "peak_memory_bytes": 22207118336}
+{"step": 88, "generator_loss": 0.1261659413576126, "generator_grad_norm": 1.8062474727630615, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.959540367126465, "peak_memory_bytes": 22207118336}
+{"step": 89, "generator_loss": 0.5089185237884521, "generator_grad_norm": 1.7027463912963867, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.154332637786865, "peak_memory_bytes": 22207118336}
+{"step": 90, "generator_loss": 0.34627798199653625, "generator_grad_norm": 1.552484154701233, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.137866497039795, "peak_memory_bytes": 22207118336}
+{"step": 91, "generator_loss": 0.14112025499343872, "generator_grad_norm": 1.4742441177368164, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.9711229801178, "peak_memory_bytes": 22207118336}
+{"step": 92, "generator_loss": 0.09152910858392715, "generator_grad_norm": 2.1761772632598877, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14867377281189, "peak_memory_bytes": 22207118336}
+{"step": 93, "generator_loss": 0.10009603202342987, "generator_grad_norm": 1.093807339668274, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15107798576355, "peak_memory_bytes": 22207118336}
+{"step": 94, "generator_loss": 0.4693828821182251, "generator_grad_norm": 1.6100873947143555, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99200749397278, "peak_memory_bytes": 22207118336}
+{"step": 95, "generator_loss": 0.06470461934804916, "generator_grad_norm": 1.6082253456115723, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.100422620773315, "peak_memory_bytes": 22207118336}
+{"step": 96, "generator_loss": 0.5110249519348145, "generator_grad_norm": 1.7262959480285645, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.03326153755188, "peak_memory_bytes": 22207118336}
+{"step": 97, "generator_loss": 0.4420318603515625, "generator_grad_norm": 1.351557731628418, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.144530057907104, "peak_memory_bytes": 22207118336}
+{"step": 98, "generator_loss": 0.2108222246170044, "generator_grad_norm": 2.424837589263916, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99123549461365, "peak_memory_bytes": 22207118336}
+{"step": 99, "generator_loss": 0.12427710741758347, "generator_grad_norm": 2.3515024185180664, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10517930984497, "peak_memory_bytes": 22207118336}
+{"step": 100, "generator_loss": 0.19606299698352814, "generator_grad_norm": 1.75026273727417, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.123385190963745, "peak_memory_bytes": 22207118336}
+{"step": 101, "generator_loss": 0.2780839800834656, "generator_grad_norm": 2.47013783454895, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.4225115776062, "peak_memory_bytes": 22207118336}
+{"step": 102, "generator_loss": 0.16892671585083008, "generator_grad_norm": 1.2550132274627686, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11498975753784, "peak_memory_bytes": 22207118336}
+{"step": 103, "generator_loss": 0.11440081894397736, "generator_grad_norm": 1.2840665578842163, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.111814975738525, "peak_memory_bytes": 22207118336}
+{"step": 104, "generator_loss": 0.5467555522918701, "generator_grad_norm": 2.2099859714508057, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16308808326721, "peak_memory_bytes": 22207118336}
+{"step": 105, "generator_loss": 0.07883627712726593, "generator_grad_norm": 1.5457334518432617, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97127294540405, "peak_memory_bytes": 22207118336}
+{"step": 106, "generator_loss": 0.07676668465137482, "generator_grad_norm": 2.3209385871887207, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.15398812294006, "peak_memory_bytes": 22207118336}
+{"step": 107, "generator_loss": 0.09865998476743698, "generator_grad_norm": 1.2412309646606445, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.189512968063354, "peak_memory_bytes": 22207118336}
+{"step": 108, "generator_loss": 0.2536486089229584, "generator_grad_norm": 2.0299601554870605, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00185537338257, "peak_memory_bytes": 22207118336}
+{"step": 109, "generator_loss": 0.6593709588050842, "generator_grad_norm": 2.1253581047058105, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.08144974708557, "peak_memory_bytes": 22207118336}
+{"step": 110, "generator_loss": 0.08989529311656952, "generator_grad_norm": 1.8564542531967163, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.007404804229736, "peak_memory_bytes": 22207118336}
+{"step": 111, "generator_loss": 0.3171427547931671, "generator_grad_norm": 1.5478988885879517, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11989235877991, "peak_memory_bytes": 22207118336}
+{"step": 112, "generator_loss": 0.36275753378868103, "generator_grad_norm": 2.262678384780884, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.159722566604614, "peak_memory_bytes": 22207118336}
+{"step": 113, "generator_loss": 0.3384782075881958, "generator_grad_norm": 1.697511911392212, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.938109159469604, "peak_memory_bytes": 22207118336}
+{"step": 114, "generator_loss": 0.1852605789899826, "generator_grad_norm": 1.8848285675048828, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.07113480567932, "peak_memory_bytes": 22207118336}
+{"step": 115, "generator_loss": 0.12310432642698288, "generator_grad_norm": 1.602861762046814, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95581245422363, "peak_memory_bytes": 22207118336}
+{"step": 116, "generator_loss": 0.0966649278998375, "generator_grad_norm": 1.1640983819961548, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12900376319885, "peak_memory_bytes": 22207118336}
+{"step": 117, "generator_loss": 0.3922007977962494, "generator_grad_norm": 1.569437861442566, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.971400022506714, "peak_memory_bytes": 22207118336}
+{"step": 118, "generator_loss": 0.06388524919748306, "generator_grad_norm": 1.3081023693084717, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12765312194824, "peak_memory_bytes": 22207118336}
+{"step": 119, "generator_loss": 0.16579052805900574, "generator_grad_norm": 1.022310733795166, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10625338554382, "peak_memory_bytes": 22207118336}
+{"step": 120, "generator_loss": 0.3187159299850464, "generator_grad_norm": 1.5530506372451782, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94981288909912, "peak_memory_bytes": 22207118336}
+{"step": 121, "generator_loss": 0.1677510142326355, "generator_grad_norm": 1.6895885467529297, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14004111289978, "peak_memory_bytes": 22207118336}
+{"step": 122, "generator_loss": 0.094737209379673, "generator_grad_norm": 1.5038323402404785, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97955107688904, "peak_memory_bytes": 22207118336}
+{"step": 123, "generator_loss": 0.21247455477714539, "generator_grad_norm": 1.9397772550582886, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.167991161346436, "peak_memory_bytes": 22207118336}
+{"step": 124, "generator_loss": 0.11005553603172302, "generator_grad_norm": 1.5294806957244873, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00644659996033, "peak_memory_bytes": 22207118336}
+{"step": 125, "generator_loss": 0.28049808740615845, "generator_grad_norm": 1.7781051397323608, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.163370847702026, "peak_memory_bytes": 22207118336}
+{"step": 126, "generator_loss": 0.4409087598323822, "generator_grad_norm": 1.4165542125701904, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.147095680236816, "peak_memory_bytes": 22207118336}
+{"step": 127, "generator_loss": 0.2021661102771759, "generator_grad_norm": 0.9934236407279968, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.019855976104736, "peak_memory_bytes": 22207118336}
+{"step": 128, "generator_loss": 0.13667267560958862, "generator_grad_norm": 1.3995862007141113, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.160027742385864, "peak_memory_bytes": 22207118336}
+{"step": 129, "generator_loss": 0.3181486427783966, "generator_grad_norm": 1.9264264106750488, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.016106605529785, "peak_memory_bytes": 22207118336}
+{"step": 130, "generator_loss": 0.14080047607421875, "generator_grad_norm": 1.609979510307312, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20154047012329, "peak_memory_bytes": 22207118336}
+{"step": 131, "generator_loss": 0.3658043146133423, "generator_grad_norm": 1.7762566804885864, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.01069092750549, "peak_memory_bytes": 22207118336}
+{"step": 132, "generator_loss": 0.19578713178634644, "generator_grad_norm": 1.3966472148895264, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.093053579330444, "peak_memory_bytes": 22207118336}
+{"step": 133, "generator_loss": 0.35997217893600464, "generator_grad_norm": 2.0859293937683105, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95857906341553, "peak_memory_bytes": 22207118336}
+{"step": 134, "generator_loss": 0.28564485907554626, "generator_grad_norm": 1.9864989519119263, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12116289138794, "peak_memory_bytes": 22207118336}
+{"step": 135, "generator_loss": 0.07675541192293167, "generator_grad_norm": 1.234764814376831, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.26844501495361, "peak_memory_bytes": 22207118336}
+{"step": 136, "generator_loss": 0.189236119389534, "generator_grad_norm": 1.7621843814849854, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.01594090461731, "peak_memory_bytes": 22207118336}
+{"step": 137, "generator_loss": 0.28312042355537415, "generator_grad_norm": 2.1078708171844482, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.24106431007385, "peak_memory_bytes": 22207118336}
+{"step": 138, "generator_loss": 0.2901826500892639, "generator_grad_norm": 1.2722467184066772, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98791837692261, "peak_memory_bytes": 22207118336}
+{"step": 139, "generator_loss": 0.11543154716491699, "generator_grad_norm": 1.2823446989059448, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.157233238220215, "peak_memory_bytes": 22207118336}
+{"step": 140, "generator_loss": 0.07825464755296707, "generator_grad_norm": 1.4255326986312866, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.18507623672485, "peak_memory_bytes": 22207118336}
+{"step": 141, "generator_loss": 0.1943899393081665, "generator_grad_norm": 2.26754093170166, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00777864456177, "peak_memory_bytes": 22207118336}
+{"step": 142, "generator_loss": 0.4914672076702118, "generator_grad_norm": 1.1965266466140747, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16710877418518, "peak_memory_bytes": 22207118336}
+{"step": 143, "generator_loss": 0.06273947656154633, "generator_grad_norm": 1.720097541809082, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.991164684295654, "peak_memory_bytes": 22207118336}
+{"step": 144, "generator_loss": 0.057838547974824905, "generator_grad_norm": 2.428715705871582, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14326524734497, "peak_memory_bytes": 22207118336}
+{"step": 145, "generator_loss": 0.16398681700229645, "generator_grad_norm": 1.3969496488571167, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.01636219024658, "peak_memory_bytes": 22207118336}
+{"step": 146, "generator_loss": 0.0491686686873436, "generator_grad_norm": 1.5501995086669922, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.13402771949768, "peak_memory_bytes": 22207118336}
+{"step": 147, "generator_loss": 0.2736506164073944, "generator_grad_norm": 1.2012521028518677, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.083115100860596, "peak_memory_bytes": 22207118336}
+{"step": 148, "generator_loss": 0.12724533677101135, "generator_grad_norm": 1.5105080604553223, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02613854408264, "peak_memory_bytes": 22207118336}
+{"step": 149, "generator_loss": 0.11375149339437485, "generator_grad_norm": 1.623902440071106, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11934995651245, "peak_memory_bytes": 22207118336}
+{"step": 150, "generator_loss": 0.18426693975925446, "generator_grad_norm": 1.4353090524673462, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0174355506897, "peak_memory_bytes": 22207118336}
+{"step": 151, "generator_loss": 0.08559864014387131, "generator_grad_norm": 1.721066951751709, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12822461128235, "peak_memory_bytes": 22207118336}
+{"step": 152, "generator_loss": 0.45589375495910645, "generator_grad_norm": 1.5163670778274536, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99076843261719, "peak_memory_bytes": 22207118336}
+{"step": 153, "generator_loss": 0.10044481605291367, "generator_grad_norm": 1.001991629600525, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1940393447876, "peak_memory_bytes": 22207118336}
+{"step": 154, "generator_loss": 0.05643880367279053, "generator_grad_norm": 1.314540147781372, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.159382820129395, "peak_memory_bytes": 22207118336}
+{"step": 155, "generator_loss": 0.09947419166564941, "generator_grad_norm": 2.2276923656463623, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.94768261909485, "peak_memory_bytes": 22207118336}
+{"step": 156, "generator_loss": 0.12572772800922394, "generator_grad_norm": 1.8656187057495117, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.157745599746704, "peak_memory_bytes": 22207118336}
+{"step": 157, "generator_loss": 0.1338098645210266, "generator_grad_norm": 1.5297601222991943, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10372233390808, "peak_memory_bytes": 22207118336}
+{"step": 158, "generator_loss": 0.10872980952262878, "generator_grad_norm": 1.290140151977539, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.12892198562622, "peak_memory_bytes": 22207118336}
+{"step": 159, "generator_loss": 0.11685019731521606, "generator_grad_norm": 1.347337007522583, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99300479888916, "peak_memory_bytes": 22207118336}
+{"step": 160, "generator_loss": 0.06638473272323608, "generator_grad_norm": 1.2787117958068848, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1345374584198, "peak_memory_bytes": 22207118336}
+{"step": 161, "generator_loss": 0.12592066824436188, "generator_grad_norm": 1.4565128087997437, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.20680236816406, "peak_memory_bytes": 22207118336}
+{"step": 162, "generator_loss": 0.17937740683555603, "generator_grad_norm": 1.124849796295166, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.09462904930115, "peak_memory_bytes": 22207118336}
+{"step": 163, "generator_loss": 0.10909775644540787, "generator_grad_norm": 0.9765651226043701, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.174782276153564, "peak_memory_bytes": 22207118336}
+{"step": 164, "generator_loss": 0.10500487685203552, "generator_grad_norm": 1.0622280836105347, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02068829536438, "peak_memory_bytes": 22207118336}
+{"step": 165, "generator_loss": 0.0975889191031456, "generator_grad_norm": 1.4407185316085815, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10222029685974, "peak_memory_bytes": 22207118336}
+{"step": 166, "generator_loss": 0.08399040251970291, "generator_grad_norm": 1.5599360466003418, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0102276802063, "peak_memory_bytes": 22207118336}
+{"step": 167, "generator_loss": 0.19089213013648987, "generator_grad_norm": 1.239227056503296, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.13632583618164, "peak_memory_bytes": 22207118336}
+{"step": 168, "generator_loss": 0.03893502056598663, "generator_grad_norm": 1.4975802898406982, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16492295265198, "peak_memory_bytes": 22207118336}
+{"step": 169, "generator_loss": 0.2590068578720093, "generator_grad_norm": 1.482961654663086, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.95993375778198, "peak_memory_bytes": 22207118336}
+{"step": 170, "generator_loss": 0.1318950057029724, "generator_grad_norm": 1.209923505783081, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.08711886405945, "peak_memory_bytes": 22207118336}
+{"step": 171, "generator_loss": 0.20269203186035156, "generator_grad_norm": 1.218271017074585, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.108256816864014, "peak_memory_bytes": 22207118336}
+{"step": 172, "generator_loss": 0.06912146508693695, "generator_grad_norm": 1.4192423820495605, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.10690665245056, "peak_memory_bytes": 22207118336}
+{"step": 173, "generator_loss": 0.35419201850891113, "generator_grad_norm": 1.2799957990646362, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.0041184425354, "peak_memory_bytes": 22207118336}
+{"step": 174, "generator_loss": 0.11835510283708572, "generator_grad_norm": 0.979097843170166, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11363220214844, "peak_memory_bytes": 22207118336}
+{"step": 175, "generator_loss": 0.21279339492321014, "generator_grad_norm": 1.367837905883789, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.087671756744385, "peak_memory_bytes": 22207118336}
+{"step": 176, "generator_loss": 0.11352231353521347, "generator_grad_norm": 1.6977291107177734, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99652647972107, "peak_memory_bytes": 22207118336}
+{"step": 177, "generator_loss": 0.25247445702552795, "generator_grad_norm": 1.2985444068908691, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14659142494202, "peak_memory_bytes": 22207118336}
+{"step": 178, "generator_loss": 0.5459132790565491, "generator_grad_norm": 1.7139537334442139, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.98044276237488, "peak_memory_bytes": 22207118336}
+{"step": 179, "generator_loss": 0.09658633917570114, "generator_grad_norm": 0.9648171663284302, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.188313007354736, "peak_memory_bytes": 22207118336}
+{"step": 180, "generator_loss": 0.11270973831415176, "generator_grad_norm": 1.477641224861145, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.06216907501221, "peak_memory_bytes": 22207118336}
+{"step": 181, "generator_loss": 0.07172924280166626, "generator_grad_norm": 1.365106225013733, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.02128791809082, "peak_memory_bytes": 22207118336}
+{"step": 182, "generator_loss": 0.364250510931015, "generator_grad_norm": 2.1649892330169678, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.16212463378906, "peak_memory_bytes": 22207118336}
+{"step": 183, "generator_loss": 0.08757363259792328, "generator_grad_norm": 0.7875414490699768, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.00282144546509, "peak_memory_bytes": 22207118336}
+{"step": 184, "generator_loss": 0.15750828385353088, "generator_grad_norm": 1.8050282001495361, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.11948800086975, "peak_memory_bytes": 22207118336}
+{"step": 185, "generator_loss": 0.1789943426847458, "generator_grad_norm": 1.7461631298065186, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.97862935066223, "peak_memory_bytes": 22207118336}
+{"step": 186, "generator_loss": 0.15961214900016785, "generator_grad_norm": 1.236870288848877, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.17013382911682, "peak_memory_bytes": 22207118336}
+{"step": 187, "generator_loss": 0.1490759402513504, "generator_grad_norm": 1.1443147659301758, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.245689392089844, "peak_memory_bytes": 22207118336}
+{"step": 188, "generator_loss": 0.1929791122674942, "generator_grad_norm": 1.2690677642822266, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.99905037879944, "peak_memory_bytes": 22207118336}
+{"step": 189, "generator_loss": 0.16703474521636963, "generator_grad_norm": 1.3049200773239136, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.099045276641846, "peak_memory_bytes": 22207118336}
+{"step": 190, "generator_loss": 0.22909918427467346, "generator_grad_norm": 1.5877889394760132, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.973697662353516, "peak_memory_bytes": 22207118336}
+{"step": 191, "generator_loss": 0.06556206196546555, "generator_grad_norm": 0.8406690359115601, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.23313021659851, "peak_memory_bytes": 22207118336}
+{"step": 192, "generator_loss": 0.052725229412317276, "generator_grad_norm": 0.9028629660606384, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.22736120223999, "peak_memory_bytes": 22207118336}
+{"step": 193, "generator_loss": 0.2824629843235016, "generator_grad_norm": 1.8263561725616455, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.986740589141846, "peak_memory_bytes": 22207118336}
+{"step": 194, "generator_loss": 0.24938331544399261, "generator_grad_norm": 1.8428717851638794, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.1851122379303, "peak_memory_bytes": 22207118336}
+{"step": 195, "generator_loss": 0.10152085870504379, "generator_grad_norm": 1.259482502937317, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.985023498535156, "peak_memory_bytes": 22207118336}
+{"step": 196, "generator_loss": 0.11434702575206757, "generator_grad_norm": 0.911045253276825, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.19030404090881, "peak_memory_bytes": 22207118336}
+{"step": 197, "generator_loss": 0.2678997814655304, "generator_grad_norm": 1.329725980758667, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.015969038009644, "peak_memory_bytes": 22207118336}
+{"step": 198, "generator_loss": 0.3913394510746002, "generator_grad_norm": 1.76265287399292, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.14803981781006, "peak_memory_bytes": 22207118336}
+{"step": 199, "generator_loss": 0.07625910639762878, "generator_grad_norm": 1.643726110458374, "temporal_rope_alpha": 0.0, "seconds_per_step": 34.157867193222046, "peak_memory_bytes": 22207118336}
+{"step": 200, "generator_loss": 0.07756521552801132, "generator_grad_norm": 1.2923520803451538, "temporal_rope_alpha": 0.0, "seconds_per_step": 33.957427978515625, "peak_memory_bytes": 22207118336}

outputs/temporal_nope/a5_tdrope_anneal_lr1e7_probe200_seed20260805_kerrigan/validation_metrics.jsonl

@@ -0,0 +1,3 @@
+{"step": 0, "validation_loss": 0.06264185276813805, "temporal_rope_alpha": 1.0, "validation_samples": 32, "validation_seconds": 22.847965478897095, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 100, "validation_loss": 0.18985189858358353, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.650352478027344, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}
+{"step": 200, "validation_loss": 0.14413261564914137, "temporal_rope_alpha": 0.0, "validation_samples": 32, "validation_seconds": 23.630066871643066, "validation_seed": 20260805, "split_manifest": "/home/jiaqi/NoPE/artifacts/data_splits/cpt_train_val_seed20260805.json"}