diff --git a/.gitattributes b/.gitattributes index ff972249c364be19897139d0429832c8b0867ef2..d0ad67d380c5dc7af7d0b06b6ecd581071051c31 100644 --- a/.gitattributes +++ b/.gitattributes @@ -97,3 +97,73 @@ artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta0 artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00007813/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/dcp/.metadata filter=lfs diff=lfs merge=lfs -text artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/.metadata filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp filter=lfs diff=lfs merge=lfs -text diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b52a6453515a5ae379c80ec08167add215d0c1df --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00015625","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"59f16f5aec7130bc28a410bb74669f878134e1297875b3edafab4caf7ff0e5a2","files":{"dcp/.metadata":"0bf44a6538d04e9ec2e002c93ef032f9da618c488b3eadc13718e8df245487a8","dcp/__0_0.distcp":"22d18487dc182949ab8270f354beda8ad7b299ddb76805e391b673c9c4b1d08f","state.pt":"45f49958fdb8800b45daff2653633db66deea3a468a66e19470124a9f96827c1"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"f1e793ec41c52c7dba5f05d83a20fddabe98738f713c4301942f589c11395889","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":4000000,"latent_tokens":4096000000,"microstep":0,"optimizer_step":15625,"phase":"base","phase_step":15625,"sampler_cursor":32000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee81a91c3d76e840504f7e2a7f87f51d351b9c86 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/base/checkpoints/step-00015625/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45f49958fdb8800b45daff2653633db66deea3a468a66e19470124a9f96827c1 +size 168490858 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0df1709d8584e973b283771d2c3a6940277839 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-formal4m-fullbatch","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/formal_4m","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"ce3557a158b8073ad07e8e1a00c61779ce7e237379eb2b3ad35132c8c2b90983","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/formal_4m","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e3bc1a18073a993b3384b33a5567312f6c886b51 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"90da573af3c8db0369df582cd32bf72f265c7ef2c0b7ab170f8f6ba96b14a05a","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"8419e64309ae8865369daebb310189cca9b4332683af3391cf730d4748e8bfd3","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00015625/manifest.json","schema_version":"1.0.0"},{"hash":"6ed75f274938edd114e198f05996706c4bc2ae2c6037b6dc4567a3df96ab207a","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"6cbca79fd0c465da4aaaff4c6a22cf82b092b9ed44b4094ce2861bc586a387e6","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"28556d43689be4e5ab9c667252eb69df1da4ece085b569726ace84f38a9725b9","created_at":"2026-08-07T05:11:02.355586+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-formal4m-fullbatch","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..1453e9b9861047f90595c7a2c73087c9de26b79d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-formal4m-fullbatch/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"28556d43689be4e5ab9c667252eb69df1da4ece085b569726ace84f38a9725b9","reason":"BASE_ROUTER_GATE_FAILED","status":"FAILED_SCIENCE","updated_at":"2026-08-07T05:26:36.968461+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..feec05c9d8fdc3cb3537f928be33850af3557515 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +b0c969f89539538ca8e26843963d582e9da1522203a8647fcf46a77d29497a67 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..604f46d54d95084696de207594a420987644b73e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5fcdd4cfae52b672c2b60e805680be4877a3c3594efc5128c72c0b0076c4013 +size 296504 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8beb9a2dd65e593749049e4a3f17843dbbeb44fb --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"b0c969f89539538ca8e26843963d582e9da1522203a8647fcf46a77d29497a67","files":{"dcp/.metadata":"e5fcdd4cfae52b672c2b60e805680be4877a3c3594efc5128c72c0b0076c4013","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"406384e5c65b176dff4d8ca0c218795d268dc749b66ab93132f1d5269385c368"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"e34b4caddd1e30ecd149c7a9d72d4c2fe57e6599ec43685535be06b5e8b258cc","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..0576a945644fad4d20cca27b2f6cc40dd0901c23 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406384e5c65b176dff4d8ca0c218795d268dc749b66ab93132f1d5269385c368 +size 168498858 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..20efe64295de3b359b8c0a0f4da3b1aa28435cc4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +656a7d0e2de8d89c98321dc46847b5430c83dc03e930c4d762da3ce91d6f7a66 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..93a5dabbda83e1d70572e74ae2ab8216d421b597 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263c4d49dd483a7bac59899d9307c3c11d2be31cac9326754c0c2c0f57f99078 +size 296498 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..db0272fe7dcfcf634712e6e74b8ac81df325070d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..946876d8b6fd9888fb792d077f5230273a226932 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"656a7d0e2de8d89c98321dc46847b5430c83dc03e930c4d762da3ce91d6f7a66","files":{"dcp/.metadata":"263c4d49dd483a7bac59899d9307c3c11d2be31cac9326754c0c2c0f57f99078","dcp/__0_0.distcp":"263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f","state.pt":"22e5baafe2fdd4a146f4852faa1a37cc8698eab2cc721b87c9fd6c722b97720b"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":100000,"latent_tokens":102400000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":100000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f9f2deed4d4a21785fdf2d7b406a93215bcd5f5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e5baafe2fdd4a146f4852faa1a37cc8698eab2cc721b87c9fd6c722b97720b +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..064d950738e92e580042d570a60447a2625ccabe --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +e34b4caddd1e30ecd149c7a9d72d4c2fe57e6599ec43685535be06b5e8b258cc diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..663e5af1790fa69adb144673a8f1c0f6b33cbc6d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6a4691a378d48cb917bb1ff213434660d49a88d6d37309178f8dd450d8835da +size 296498 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..9d640610a7400ef73f616d5b1c490e2cf1cc852e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"e34b4caddd1e30ecd149c7a9d72d4c2fe57e6599ec43685535be06b5e8b258cc","files":{"dcp/.metadata":"e6a4691a378d48cb917bb1ff213434660d49a88d6d37309178f8dd450d8835da","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"5bc66fa76c2e48dff8bf67ccc5d714d3a866d3c97bf79609825d7bcbfe10f960"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"656a7d0e2de8d89c98321dc46847b5430c83dc03e930c4d762da3ce91d6f7a66","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..828da9b70584fc36b9e6c5f593c28f38e768c041 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bc66fa76c2e48dff8bf67ccc5d714d3a866d3c97bf79609825d7bcbfe10f960 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..15d1d1c9e8c5e1ef8baa52263c23c309d96817f4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-nosync200-batch1000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"cc2cfec0ffa7713c69f9932a3ce1785eaaabf50d26405e91469829b8c15b757b","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..1e8d91ce5f92ff04982e52d2b562a51ef4c27ef5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"521d9b82197e8b46a2d980ce232e73f02c4d36f8c36573cb3f2d05cd9a1718bb","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"dfda0b3d74049fc178b1fff77938997ce24803b516f835e8aeaef529be29caa3","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"9f4a3bd78c512d43270d7d9130f0a22793864f0ac2784f598bae0db679d8cab2","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"5183bb961b9ce47ba50b5d307b5cd783995a12c9e1ffd05234009a54d411f272","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"880a38edc0d1a6ced983df88033c039b1eb49abcf9f9507b789f421579dc185f","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"753ba930fb09348142d5331c1a5fdf59fc6b64b76da17f2ea5386f7d65093e24","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"53fd62885e3c576f857c47dca91981e5ad9e19d833b30215afd98f1fec75d570","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"09ab2a90c42b9f81cee34424bff3737415e94518922d2256ca3dd4692854c072","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"1d9dfbf6a18945a361f04bcce1ef5d825def337d10133b94a9ee64480066508f","created_at":"2026-08-07T08:11:16.520742+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-nosync200-batch1000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..00dc7e0d7d4f29933d6b2097c32f25cd922b3138 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nosync200-batch1000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"1d9dfbf6a18945a361f04bcce1ef5d825def337d10133b94a9ee64480066508f","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T08:11:16.804569+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..91cbdb355125a45eed200a7b9905ca914db836da --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +e26b841ab4af97ce641e60d47c7a2b0b0545176b9e285a60151ad905f87ff6c4 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..56e568f8d14d03c6fbf6f53f026773a51c6ff77d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d132d2b57758e886bf63d9e9807b8672279299a06834de28b0069ef7a71ff50 +size 296502 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..00f2ce9bd91fa7d9129ef7ae76dde188db1ed0a1 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"e26b841ab4af97ce641e60d47c7a2b0b0545176b9e285a60151ad905f87ff6c4","files":{"dcp/.metadata":"5d132d2b57758e886bf63d9e9807b8672279299a06834de28b0069ef7a71ff50","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"0f06f8c80dc7f99ceecc88454b0bb2e2fac081602b6903f484a4a0d57cebfbb5"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"f70c283cba852e6cc69717f0f6de13b7de359bbd2fc3d34293e98d60e4d58295","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..6907dbf42e3c5064244d01c8a5c39c4e67866d3a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f06f8c80dc7f99ceecc88454b0bb2e2fac081602b6903f484a4a0d57cebfbb5 +size 168499114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..32b513d9719268d07c4fdb29d51b6d36a73f0c35 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +491cfdc2a1293b902f780998db6a4caa278db47e681c54124a2fec0975fa7f5f diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ee97aaf06d39b20f7b1f249cd8fc110a98cdffc0 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17edd1b9bbeda95db0446d89ffc877cd26c068f7a08d6df756d836438b51554b +size 296496 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..db0272fe7dcfcf634712e6e74b8ac81df325070d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a0cdb3c12770e1e4c6ad56fd789d7eab3dbc9a51 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"491cfdc2a1293b902f780998db6a4caa278db47e681c54124a2fec0975fa7f5f","files":{"dcp/.metadata":"17edd1b9bbeda95db0446d89ffc877cd26c068f7a08d6df756d836438b51554b","dcp/__0_0.distcp":"263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f","state.pt":"31f53017152d3e32da70eb059848df3dde7ff2ae58fa9d904229ebcf6e66485f"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":100000,"latent_tokens":102400000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":100000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebe44b36dea41216484911211a3a1b31558a92ff --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31f53017152d3e32da70eb059848df3dde7ff2ae58fa9d904229ebcf6e66485f +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..35609e1621e087e03148ef303dc71e22bdaa0711 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +f70c283cba852e6cc69717f0f6de13b7de359bbd2fc3d34293e98d60e4d58295 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..34771c8266b0c35c2dc015fbb18620d580f7f08c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:726c22a54fdb64e3855550759236c781ca0d5ab0b8979033f8859d480efc77b4 +size 296496 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..197d9f552b5ae43f21a7846d5c44179592029abd --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"f70c283cba852e6cc69717f0f6de13b7de359bbd2fc3d34293e98d60e4d58295","files":{"dcp/.metadata":"726c22a54fdb64e3855550759236c781ca0d5ab0b8979033f8859d480efc77b4","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"f7eef57c77ac4685861066db5bf0c2efad3671cd2362536e63920ded21d32b3d"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"491cfdc2a1293b902f780998db6a4caa278db47e681c54124a2fec0975fa7f5f","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..a22e6058ad261b711cf0244d148bc618e44c785e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7eef57c77ac4685861066db5bf0c2efad3671cd2362536e63920ded21d32b3d +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..792bc27895267258dad39341a2614885a568a03c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-nsys100-batch1000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"cc2cfec0ffa7713c69f9932a3ce1785eaaabf50d26405e91469829b8c15b757b","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ac4d76de72f4e3f9a7494ba2a3ef191b0cee78a7 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"a609ade5352d66fab474125b6b1f5eb9445bcf9aa84c3df52089f56d051f25c6","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"40d9c36b99f52bd5c52d54776806d8a065d3902ddc1409b9f2af52fa99215652","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"1d81737b2c255ec5d4e96b387e64ea274e473f05b8c06d1df4c133afd56c4e78","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"5183bb961b9ce47ba50b5d307b5cd783995a12c9e1ffd05234009a54d411f272","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"d85c008a8de3eb0dfc9207edc615a23614bdcc69675923aad1f10bbd04af48eb","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"ccb85c9ffbf48013242132e6d1a27013e2ae3aec37552bbb008b676a324c55d7","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"420a01f3482ec5b7c51ff3f83c298ee27234b31b9d9f4cd57c6ca32f4f6ed02e","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"eea5705f122676b591892482ba41fa956952cda2260f61e590d1ed070de4ed29","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"049073201163d6c4cb82143db55f1abda74459115af775c0983efb4d4fd1e1ef","created_at":"2026-08-07T08:23:18.272068+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-nsys100-batch1000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..a0159041eadcdc1ec1acace6687fc1e6b0b6ace4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys100-batch1000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"049073201163d6c4cb82143db55f1abda74459115af775c0983efb4d4fd1e1ef","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T08:23:18.645735+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..2bed29ce8f43a0aa977192f21c2e5b3206f9dadf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +ac4c577e75586773fc50cd0578e898b0845d7644ec11784d427ec004403ce014 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f9d832e6b41b38c8f490d4ae7b01144b89ac7806 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73d7ecf06dfb57f04714375dabc5448dcaf7b06823511b53fab4b3db2c06fede +size 296501 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4015c7b328338f9512ab4788624680b4eb753355 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e5134b1b7d2f4d1597d5fad576639d83e354ca86c6ac7cee1453d44959468f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..84a2acdfbd60b0c90b7a954ac087936423a2e683 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"ac4c577e75586773fc50cd0578e898b0845d7644ec11784d427ec004403ce014","files":{"dcp/.metadata":"73d7ecf06dfb57f04714375dabc5448dcaf7b06823511b53fab4b3db2c06fede","dcp/__0_0.distcp":"a8e5134b1b7d2f4d1597d5fad576639d83e354ca86c6ac7cee1453d44959468f","state.pt":"dfb149fc1feb6a87ca09a2844f10c28302085048e46cb4d0b2b53a9719721626"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"7a035664561e3c5a60dcc8a5d19413e430764ffa1b25294f31dd22878680f662","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":20000,"latent_tokens":20480000,"microstep":0,"optimizer_step":20,"phase":"base","phase_step":20,"sampler_cursor":20000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e05dfcd10cc6baef78d75c100666601ec8acf3a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfb149fc1feb6a87ca09a2844f10c28302085048e46cb4d0b2b53a9719721626 +size 168499114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..723378d95526b76b6148eb3bb0092aaae4d261b1 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/COMMIT @@ -0,0 +1 @@ +7a035664561e3c5a60dcc8a5d19413e430764ffa1b25294f31dd22878680f662 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d59e969101b79ed367f407a508e05a7bf427722c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c4aa8137c259349d13a1149ff7d1421f070c08f54b2b90f0bd8e0a2ff57f87 +size 296495 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4015c7b328338f9512ab4788624680b4eb753355 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e5134b1b7d2f4d1597d5fad576639d83e354ca86c6ac7cee1453d44959468f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..9a72d5ff53dab01c28d4a7f2ecb3bf5e938ef404 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000020","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"7a035664561e3c5a60dcc8a5d19413e430764ffa1b25294f31dd22878680f662","files":{"dcp/.metadata":"99c4aa8137c259349d13a1149ff7d1421f070c08f54b2b90f0bd8e0a2ff57f87","dcp/__0_0.distcp":"a8e5134b1b7d2f4d1597d5fad576639d83e354ca86c6ac7cee1453d44959468f","state.pt":"da06e23de89e1d046f2ea090ac1fce338cc5cc7f5786f4981a098208c3b39a9d"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":20000,"latent_tokens":20480000,"microstep":0,"optimizer_step":20,"phase":"base","phase_step":20,"sampler_cursor":20000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..0014b90f9ebd3be5c2c6552c99e3d04f77885ee1 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/base/checkpoints/step-00000020/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da06e23de89e1d046f2ea090ac1fce338cc5cc7f5786f4981a098208c3b39a9d +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..282795bfbc00ffd20437f9d6678e23f795b87bc4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-nsys20-batch1000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_20_batch1000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"5b5a3a751fdbe798be93b49633d5e44ab6d05cdd902234b6dfb0494013a2cda1","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_20_batch1000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..275d0ef951e4b4267976857e860edac164dc4378 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"dc2a4572de4af7e6568ca06b3f55d1de2fba966fba4689eb955b7ba71cdbacd2","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"8fb4c63c768f37442afae02e994585ffaaf05e8313fa5b8a71f6c8962838ee59","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"f77ee3c0953a1e1097f6b8aae2844e542b5e8f1a8050ffb16ab25503fb8eeb2d","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"56ce3b9cb6ac3166955a19ea650bf4e3ac2ee72acb17f5b2c1e2a2f2b0836c16","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"11dba6178bce20948d78ca9bd65d87b2f6cc6f35dba2d91f14ed552efd0e96cc","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"bc58ee6224f97dde8ab3c47b1852b33f81801631425e515a52c36af8b9fc70cb","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"4435088e21a2a6391c9fac70146481c20d8b5d0ed5cabb6c3b74e886bfba877d","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"eea5705f122676b591892482ba41fa956952cda2260f61e590d1ed070de4ed29","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"dac5e180bca5fc6cba8b7e62a7b55e265dfb05786b7d0dc4236b5f48cc19f7a8","created_at":"2026-08-07T08:20:23.424455+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-nsys20-batch1000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..13db55f5ba97e0f8d7ffb603d6cab08a97eb9dad --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-nsys20-batch1000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"dac5e180bca5fc6cba8b7e62a7b55e265dfb05786b7d0dc4236b5f48cc19f7a8","reason":"BASE_ROUTER_GATE_FAILED","status":"FAILED_SCIENCE","updated_at":"2026-08-07T08:20:23.797515+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..768785003d658034c8e4f1d37913dea6202a4fcd --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +650c151c4cd63b9d16dd9310716022fb960908ca1c8693235cc11b1e927397f9 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9ba9b652451932c1eefbc92000f461cd55392671 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67aea5f82958d0993fe0c49c5960e8537b9af1eb51ffc19ad2b3f0d24e0f24ad +size 296507 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..75a571e52031044bf0e13359a1a12b3c42b66a08 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f99af0b384932feaaa87fa43e95b51da22b26e00bdfbf04dc3d8a4c1fa5f7a +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e1557348c4d3e6bc41ef98a86a555dedcc016e7f --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"650c151c4cd63b9d16dd9310716022fb960908ca1c8693235cc11b1e927397f9","files":{"dcp/.metadata":"67aea5f82958d0993fe0c49c5960e8537b9af1eb51ffc19ad2b3f0d24e0f24ad","dcp/__0_0.distcp":"64f99af0b384932feaaa87fa43e95b51da22b26e00bdfbf04dc3d8a4c1fa5f7a","state.pt":"49ff36a1a139d100cee045056e6a6a8434bb88a73cae8055c2923d3c66725af0"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"5412307375e4f0dcd9f15a12bbdcbc799cb0e3893916582a8bcd835112e9eb10","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":160000,"latent_tokens":163840000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":32000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d8555740d46ca2b6c8fc0e766fd385dc3960e8f --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49ff36a1a139d100cee045056e6a6a8434bb88a73cae8055c2923d3c66725af0 +size 168498794 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..d5812ad9f8769ab1afc15a7d27315f6b4e7c1d01 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +5412307375e4f0dcd9f15a12bbdcbc799cb0e3893916582a8bcd835112e9eb10 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5d808994ad61e97a5e35802d9c38debfb86b8eff --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67022dbfcee3160fcc29aa1f914937340e52761ee3f61105cb445784111b35c9 +size 296501 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..75a571e52031044bf0e13359a1a12b3c42b66a08 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f99af0b384932feaaa87fa43e95b51da22b26e00bdfbf04dc3d8a4c1fa5f7a +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..7b2cf768c930c4c65bbef80d06bf5c23a99c07b9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"5412307375e4f0dcd9f15a12bbdcbc799cb0e3893916582a8bcd835112e9eb10","files":{"dcp/.metadata":"67022dbfcee3160fcc29aa1f914937340e52761ee3f61105cb445784111b35c9","dcp/__0_0.distcp":"64f99af0b384932feaaa87fa43e95b51da22b26e00bdfbf04dc3d8a4c1fa5f7a","state.pt":"a9745ad6e078229aa63cf235ed694bc9ca24e5e7eff6c6fb4adb002325c3259a"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":160000,"latent_tokens":163840000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":32000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..690be13dd96e8928626e010a464d3207543407ab --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9745ad6e078229aa63cf235ed694bc9ca24e5e7eff6c6fb4adb002325c3259a +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..729779f32f497bdb9df43bde5c6ef3efde873148 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-optimized100-batch1600-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_100_batch1600","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"68b99e450de4ceb073c9d962382008414a049b80826dbd7d8284296e05529c4a","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_100_batch1600","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..5c835b3e12753dfc7d6365075fde5f6a0f1ffb94 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"6f70fb9d12e758814195c5c0e974f95065b6414280cf207884065aa8c5fdd18f","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"ef9ebb1b6b09b44efab4bbab0d9a042e3ab5b61380f17c0c6e7eeb591f098db9","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"fce7ed373b20a4eb52f3c4c991c68999efed639912d1038fe901ae510ea2866e","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"1292dc8e2095d90db9e7646dd00165f7b3f9d4ff55590f625ef8214289d46934","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"9a6a255fd9ce80308f1e09714cce4dbe1d2967ae3202c4ab336dff06f3cb20de","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"5728b30aca33d745db82b3bf058a39818c75465f9806bbd0145bf5a608a1a3e9","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"8409c9d790b1d9bd5408d65e5e00beca5277cf36687ff327d8af46f8fde4a64c","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"f79b63694ca616fb3e101c1d25141f1f43fd0a8e8d5fb6f30085a5af3683d679","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"a39aebcc587b801e125732598de32b2d52cd54cd7ff853b161fc1c4e645ea94c","created_at":"2026-08-07T07:31:37.960697+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-optimized100-batch1600-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..f515a4e14adc4b2fbd494ab15c30f1cb7f40abb6 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized100-batch1600-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"a39aebcc587b801e125732598de32b2d52cd54cd7ff853b161fc1c4e645ea94c","reason":"BASE_ROUTER_GATE_FAILED","status":"FAILED_SCIENCE","updated_at":"2026-08-07T07:31:38.291013+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..f36f669bcfc695cc413df4344f037ee2e7f575bf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +76c8ca03587c5d304b29a29c1534d94f45aaac64f0b9ed4bed3d5050092dd423 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2de88f2eb6209522c92784fbd4aaa2d8d75c17be --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed8ec2576443d5b06e94300c23ee0c8146dbadeef521c7ef64d33426ea72ec5 +size 296507 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..db8e11bca180ad612a4f36654b05123fbc6158a5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"76c8ca03587c5d304b29a29c1534d94f45aaac64f0b9ed4bed3d5050092dd423","files":{"dcp/.metadata":"9ed8ec2576443d5b06e94300c23ee0c8146dbadeef521c7ef64d33426ea72ec5","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"0f7ba7661c4681874dc19ba8cbcfb3aed681464da53f45280a6efe8dcf9217c4"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"7be1975b61888c0bb1a566997ecaa32c7a6cf2fcd4fc7e97bb4f994abae197f5","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c2af9627127f3eaf6c1dddeba5100141085b366 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f7ba7661c4681874dc19ba8cbcfb3aed681464da53f45280a6efe8dcf9217c4 +size 168498794 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..037cc87cda18d06ca91b714caf44691a5ba01218 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +4099164be86066b01b9bdaa537d9df843a1cc218953c79106c135a7303477072 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..11ee574c75be26d2ff0d8bb1384446b42ec986c6 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ef30a11220f975a90d7149ab55c8e73cc96d741b199a397359ff5f1e2cdd9e +size 296501 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..db0272fe7dcfcf634712e6e74b8ac81df325070d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..045266bdf5a83f945302d1a2226494ec929b3f4b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"4099164be86066b01b9bdaa537d9df843a1cc218953c79106c135a7303477072","files":{"dcp/.metadata":"f9ef30a11220f975a90d7149ab55c8e73cc96d741b199a397359ff5f1e2cdd9e","dcp/__0_0.distcp":"263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f","state.pt":"748adab07c12b1b3571e54af57e76ce2f599c04d1b694768266dee00334507e3"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":100000,"latent_tokens":102400000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":100000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..0cc117825ec8934cb963b9de689c25d32b34904c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:748adab07c12b1b3571e54af57e76ce2f599c04d1b694768266dee00334507e3 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..ecd7c378fef5f75c616ba28fe5d8fca3a0398b54 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +7be1975b61888c0bb1a566997ecaa32c7a6cf2fcd4fc7e97bb4f994abae197f5 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5bc9b86aa18091926d41c4199125ceb499e55bce --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c662b046fa71d6f94eca0f3f9e80914f8fe6bb6e847d18e0f33d76168def6b04 +size 296501 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8ea00644b5ca46e6eae428cb69e1d89fc5906f46 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"7be1975b61888c0bb1a566997ecaa32c7a6cf2fcd4fc7e97bb4f994abae197f5","files":{"dcp/.metadata":"c662b046fa71d6f94eca0f3f9e80914f8fe6bb6e847d18e0f33d76168def6b04","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"a5401f0377fd6e66e7b07eeaca8792bb841d19a559cd5390a390f318a39da3fe"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"4099164be86066b01b9bdaa537d9df843a1cc218953c79106c135a7303477072","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..ada206052bd96a37cb90070e7280cc7c5a69afca --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5401f0377fd6e66e7b07eeaca8792bb841d19a559cd5390a390f318a39da3fe +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..ea8e789d88349179e7b4bb1b799ca2e3cf7c016e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-optimized200-batch1000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"cc2cfec0ffa7713c69f9932a3ce1785eaaabf50d26405e91469829b8c15b757b","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ae1e8e486ee0ef1479702a0f5a28e7f531e4b8d3 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"14a5289924158b550d9696e483fe4de438f99caecbbc89a181f0f688b7d5e01e","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"f6a027d8e984f9076863aa16e0c6aec8a439b0bff6db03cc6a899f985ba00773","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"4d6c100abe34416b954e8cb89eb7c4310bd6352fc42402c063d2b1795675b35b","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"5183bb961b9ce47ba50b5d307b5cd783995a12c9e1ffd05234009a54d411f272","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"47db5ea0465a156c39b647212a407058b4965e1c0468e2f3b83293611adec13e","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"edfef01fa7be8653ba933a3bc82289f6b17a9c3e224eb8968fdfd5ec241004a9","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"315eae1489437f2af703c7877d28c2ba7ad8d548b782744b2629feb645aae7c2","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"f79b63694ca616fb3e101c1d25141f1f43fd0a8e8d5fb6f30085a5af3683d679","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"d39736f2a7db4c4d509f7054006857c5ebd3d5f2f4eb69df46ed4a536dac09ef","created_at":"2026-08-07T07:34:11.885814+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-optimized200-batch1000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..eec3639e3959b24dff75f4ea775407ddec69ea06 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch1000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"d39736f2a7db4c4d509f7054006857c5ebd3d5f2f4eb69df46ed4a536dac09ef","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T07:34:12.156366+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..6c66ce02ff7eaea6525fb8fd944a9575bcee357b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +fe568835733c6c16492df66cf3206def14b44669afc2ead36e92fb1e33775567 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4311f0513fcd6530bea594dc7b1d8d07cb1be04b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac4d0c4eb49793b44011f8c37c8040461867ede4cced4f75a80b06e31d5acb09 +size 296506 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6b301749f55d5c8a28712e3a872907c8ca4b7778 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bfa50fbe7d97860e4396b1207961a20fd0f510af355ad353b83b5ddf356220d +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d446d0f9fde599d7eb3a7d55b298b0c8acdb072b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"fe568835733c6c16492df66cf3206def14b44669afc2ead36e92fb1e33775567","files":{"dcp/.metadata":"ac4d0c4eb49793b44011f8c37c8040461867ede4cced4f75a80b06e31d5acb09","dcp/__0_0.distcp":"9bfa50fbe7d97860e4396b1207961a20fd0f510af355ad353b83b5ddf356220d","state.pt":"b8aed5d0aeb5b76d9389fa48b66a6bd2fc71e9bc94490d866f62868293f43186"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"c4675283f40f9ca081f7eaafcaebd739fa299ca9a9f89564126d5197b22b1ebd","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":128000,"latent_tokens":131072000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":128000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..74e06054bbffdc47fa6c5494b48616bd5e7361bc --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8aed5d0aeb5b76d9389fa48b66a6bd2fc71e9bc94490d866f62868293f43186 +size 168498794 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..20a5657ac35f6b372c1fc0d3571b16cb4c100a91 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +ef49f84140e559089bdc6bdc745c5036e634913ae104d216413e01b499d4ae78 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..48d7364f4a603b9f58bb3e82def56ae5f4fa6b85 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:709b48a94d0e7ca4907f8f4fed3164ed6d03f4bb633adebdef67d244ff58e7c5 +size 296500 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..44672711870a5904d660fc6bf5eafb6c1f04186b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10d55822eed78bda4a0be36e5e3edcaedb11802fef8f3cdbc12c6c4da15a1b08 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ea916c1f45c5e2ec8e5c99b7ea013be9b22f6988 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"ef49f84140e559089bdc6bdc745c5036e634913ae104d216413e01b499d4ae78","files":{"dcp/.metadata":"709b48a94d0e7ca4907f8f4fed3164ed6d03f4bb633adebdef67d244ff58e7c5","dcp/__0_0.distcp":"10d55822eed78bda4a0be36e5e3edcaedb11802fef8f3cdbc12c6c4da15a1b08","state.pt":"ab59186c3fb0259564adb93996d3689e73bccf4373b4cf86857e0753948d8668"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":64000,"latent_tokens":65536000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":64000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff02a74b32a31fe5311b17309c4fa4b22613152f --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab59186c3fb0259564adb93996d3689e73bccf4373b4cf86857e0753948d8668 +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..924d82fc4f0af78cb141fa086d360feb7a8089e1 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +c4675283f40f9ca081f7eaafcaebd739fa299ca9a9f89564126d5197b22b1ebd diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..35e3db25b849973d609d3314755ee938f99e0714 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4d6c48804df5636a6d03487d8dc181247b4e3c507b022979f7aecd1e9fa3bf +size 296500 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6b301749f55d5c8a28712e3a872907c8ca4b7778 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bfa50fbe7d97860e4396b1207961a20fd0f510af355ad353b83b5ddf356220d +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..5fa58e2105ee4460306ef2a69763652bcfa154df --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"c4675283f40f9ca081f7eaafcaebd739fa299ca9a9f89564126d5197b22b1ebd","files":{"dcp/.metadata":"2a4d6c48804df5636a6d03487d8dc181247b4e3c507b022979f7aecd1e9fa3bf","dcp/__0_0.distcp":"9bfa50fbe7d97860e4396b1207961a20fd0f510af355ad353b83b5ddf356220d","state.pt":"7cf9ce108f77969efca8323cb717b9ec7d622b1b4c47a769104fb5c2fa46d159"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"ef49f84140e559089bdc6bdc745c5036e634913ae104d216413e01b499d4ae78","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":128000,"latent_tokens":131072000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":128000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd3602d976933432ae0f0a9d33e5fe9090257fc9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf9ce108f77969efca8323cb717b9ec7d622b1b4c47a769104fb5c2fa46d159 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..f7ae062cdaf5d8bbe33347affc4cfa591c491aef --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-optimized200-batch640-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch640","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"b090c2e581b7c77403ca2a82ca9fb2498a82760beb92b7e227b53e589e29e30f","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch640","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..282b4e66ccb711d250046208b9fc401e442bdf33 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"e9302981fb9eb3972eb9a958fe60eab6e4d668d083f0778948f999bf930ff100","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"bc967da22dbfe85f4d03852a6d10514a283edbf19b6b6fec180db05ca5b3c132","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"788a2a7719e93eb045ccb219e29e42e148e4c82858ee67b4f72d82a9db192c8b","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"39e601ceb271737b30d54a7a2c7aee86b657007a60c686710e7e7bd8dd600a0f","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"a355736bfd4cce83ed960979bdfc2b4d913f18521a48da5549c7edeb2855664d","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"f1b50889b13e4314a6e56a4eb754903e48c6b5f745272d82fbc47d86bad0c183","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"3ad6d569db1280ade3053c175b7dccee24a94783d520cf9a319811f2ca16f774","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"f79b63694ca616fb3e101c1d25141f1f43fd0a8e8d5fb6f30085a5af3683d679","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"a6590928c094e5de433bd9d98543c920c99464e2dbc987ced4dd3e5f6bcdf453","created_at":"2026-08-07T07:29:21.043048+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-optimized200-batch640-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..601fa754fbbfaf0034be2a3445af2c26024c25f5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-batch640-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"a6590928c094e5de433bd9d98543c920c99464e2dbc987ced4dd3e5f6bcdf453","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T07:29:21.332490+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..bb598a64f133fb3de3452a9a689fab3b2991aa03 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +e5364cb7e3234632ba7a4fc6c69d89236e7503a30c1430a7de35344a39c039e1 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e3d01537da6c68b8e0329f7898e56da0d2e18ff9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2fc2a2c666f76483b0c0f5c0cd3367b69b437b81f432c6f6e5401bc8e2a7a3 +size 296509 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bd60b742a7881893f2f14c5901eef290610102a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8e6c22f5ccdbdc00ebb11d5b226db45157d64bb8 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"e5364cb7e3234632ba7a4fc6c69d89236e7503a30c1430a7de35344a39c039e1","files":{"dcp/.metadata":"eb2fc2a2c666f76483b0c0f5c0cd3367b69b437b81f432c6f6e5401bc8e2a7a3","dcp/__0_0.distcp":"4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c","state.pt":"b91bef8f29449b9c8486c763b27afdb28502cdf5eae3788f848f987b6789d682"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"198a19e1516bbb09a97313417adc7599ac78319e76c32bd0cd72676d2c37bcdf","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf5406a0057058fdd1a7a62f10017ed551fab7ac --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b91bef8f29449b9c8486c763b27afdb28502cdf5eae3788f848f987b6789d682 +size 168498474 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..2d89bbba5e8c1075962c0786bd586160329ca626 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +fb5096b88ef8541cabbd6cc85f0df1c39ef5c50d5208720485d4247e40d01ec9 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..73b131e6f06826db8e7a30db8b80314f23758d53 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca5b0eb6520c7a0329fa4d993474069b4e4c2b82fa828b122c506fd99b79b434 +size 296503 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fd70a0594390f18be9403745568c7e21a3c931f5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c7dfadb6c580561f94b17729dd752e2eeff0e31a49dcbd70296ce4a8927319f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..94efc19ff5c4692c88033d8a63156ccc339ff26c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"fb5096b88ef8541cabbd6cc85f0df1c39ef5c50d5208720485d4247e40d01ec9","files":{"dcp/.metadata":"ca5b0eb6520c7a0329fa4d993474069b4e4c2b82fa828b122c506fd99b79b434","dcp/__0_0.distcp":"2c7dfadb6c580561f94b17729dd752e2eeff0e31a49dcbd70296ce4a8927319f","state.pt":"597a7c251fa88062b353284bc81495cda2f528f9120eab9c29b39b0306d8f952"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25600,"latent_tokens":26214400,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":25600},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..1105b27d67f800a79f9d0e69719fb6983a714226 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:597a7c251fa88062b353284bc81495cda2f528f9120eab9c29b39b0306d8f952 +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..9994f02cfcb036429f3deb6454e43f4f0f7e6230 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +198a19e1516bbb09a97313417adc7599ac78319e76c32bd0cd72676d2c37bcdf diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1ae831a79c9c85b60336a27f3edeb1097c0c850e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2acefbaa0488d14a7e85cf8970441a899f71203bc9341f8bc7b8784558395881 +size 296503 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bd60b742a7881893f2f14c5901eef290610102a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..07e59471e5d380eac4052c0927cb8b414048ae1c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"198a19e1516bbb09a97313417adc7599ac78319e76c32bd0cd72676d2c37bcdf","files":{"dcp/.metadata":"2acefbaa0488d14a7e85cf8970441a899f71203bc9341f8bc7b8784558395881","dcp/__0_0.distcp":"4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c","state.pt":"4f96b3676fad7d0c0344cbe6726cc20f4c935a6ccb3afab4da2d1c69e920154b"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"fb5096b88ef8541cabbd6cc85f0df1c39ef5c50d5208720485d4247e40d01ec9","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..441e5ced15928afe5f3445748248d12f1ebb597d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f96b3676fad7d0c0344cbe6726cc20f4c935a6ccb3afab4da2d1c69e920154b +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8ea6d07c5265d13a283a174e4ed7238559f7dd --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-optimized200-routerstats-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/continuous_200","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"b4b35c3a6a0e4cf498fda9f5fd5f66c5a7490c42dddc24446759bd2ab105a1ef","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/continuous_200","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..829dd044ee3ae724dd6b992fc35300caf0c814c4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"c87d3f39b767084a287bf41ed449d00ab2a3579fece1c737417d3972707c194d","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"2b20b3117caa482bc2fc69f4198340fdea9840dcff26704ce22bb81cc16884b6","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"b366153d26927d14ceae015c0d0fe3c8d64c720c5b6fa6e81832f68af1d5a14a","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"ad99fcbc68da63dfcfbf87ed8fc3c02294f3e11c760f038291b71836fe37b4b8","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"d9081c4b74a9de14f27b707703549619e3bc0a9e9db0628f057c08b2166bb825","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"ecb4c647fb5e1ab1092a96235433db7951ce5bff04546a92884ea79682e6ca01","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"15cc0165dd8f47035b4606d5263c25b06fad0718c3fcbb3cc8f467793aea0156","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"c396538d0007479e76487cca48928a5e9dd9a22472f729e2d8ba6e1c00ffe6a1","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"37a78cca4569def0ad480e72e580fc1ef2dea4e6e0139a0fbdf209d5095ff0db","created_at":"2026-08-07T07:22:20.333260+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-optimized200-routerstats-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..69f2ee906b88c299f9f37a45d3c7d0b180591ca8 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"37a78cca4569def0ad480e72e580fc1ef2dea4e6e0139a0fbdf209d5095ff0db","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T07:22:20.573554+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..9d331a72ba8a82dfd00d4765168054f511538f3c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +6bc63cd7f29217514ebe621aa9a6031f486ea0742dfbd4afb2f1c089000212e0 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..7bc320957836e383ab56802f46b7eaf5abe085bf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a265d70343f279f5b0f878df69d17af8a1286814fc3ba69fe1cf7dffacceb8 +size 296509 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bd60b742a7881893f2f14c5901eef290610102a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..19e580554572dbf3423bc4fd12b63c29dfe7eca0 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"6bc63cd7f29217514ebe621aa9a6031f486ea0742dfbd4afb2f1c089000212e0","files":{"dcp/.metadata":"92a265d70343f279f5b0f878df69d17af8a1286814fc3ba69fe1cf7dffacceb8","dcp/__0_0.distcp":"4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c","state.pt":"cf80578489f85433f6d72666e61ccdb0d4f066ee48dfe64b0923f59ac7e92495"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"6cfcf88d0792c9cf79843cb86170bfe5b7f51ffe133256a31b33ecd0be5c2732","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..011208539e25a1f5865c2e9c63e6db33e5a0e11b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf80578489f85433f6d72666e61ccdb0d4f066ee48dfe64b0923f59ac7e92495 +size 168498730 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..219635f092e64e5ec044d9715cd9b4e7a922a51f --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +7a5ded4bde9f210f08f8c091b13d754637f9a4f8c1d45621b4ab84a38fde8a52 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..53bd304b170ec7d7da68baa4997c6f6b74caaec7 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b08b9f6779d6a9ce6350096b492659e37001aa1ef30ca5ccea494bbaa8479786 +size 296503 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fd70a0594390f18be9403745568c7e21a3c931f5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c7dfadb6c580561f94b17729dd752e2eeff0e31a49dcbd70296ce4a8927319f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..09c72f9790dba522d524a5e8e778e046094c0735 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"7a5ded4bde9f210f08f8c091b13d754637f9a4f8c1d45621b4ab84a38fde8a52","files":{"dcp/.metadata":"b08b9f6779d6a9ce6350096b492659e37001aa1ef30ca5ccea494bbaa8479786","dcp/__0_0.distcp":"2c7dfadb6c580561f94b17729dd752e2eeff0e31a49dcbd70296ce4a8927319f","state.pt":"d6be00bb234eb2eec548c9d86b5e0effaee361ed39dc1269fb41ea18f5c45e12"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25600,"latent_tokens":26214400,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":25600},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e27cb8610b431eb971b677cf8e06fdfd0dd2057 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6be00bb234eb2eec548c9d86b5e0effaee361ed39dc1269fb41ea18f5c45e12 +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..fb8199f5f2e7a296029eb67093fd67fe06e3b855 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +6cfcf88d0792c9cf79843cb86170bfe5b7f51ffe133256a31b33ecd0be5c2732 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2c627071b3b911ba6377da43f09a6b4e25043adf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fdfa9990efc9ca58644e89c20002f2a1bf379846b34fdcb6357aa2a5088e6be +size 296503 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bd60b742a7881893f2f14c5901eef290610102a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..2fcb86147d00b3a16803f3c99e12f231ac2bffca --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"6cfcf88d0792c9cf79843cb86170bfe5b7f51ffe133256a31b33ecd0be5c2732","files":{"dcp/.metadata":"3fdfa9990efc9ca58644e89c20002f2a1bf379846b34fdcb6357aa2a5088e6be","dcp/__0_0.distcp":"4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c","state.pt":"64f10de8ca296d307d35fc467976833988eccea858e41bee52c89ebe337d248a"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"7a5ded4bde9f210f08f8c091b13d754637f9a4f8c1d45621b4ab84a38fde8a52","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcb5ffee5e38527bd8b7314192432b84870b34a9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f10de8ca296d307d35fc467976833988eccea858e41bee52c89ebe337d248a +size 168491114 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..20f5adbe18226022ea90736811cbade8e2ceee55 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-optimized200-routerstats-20260807b","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/continuous_200","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"b4b35c3a6a0e4cf498fda9f5fd5f66c5a7490c42dddc24446759bd2ab105a1ef","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/continuous_200","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3a94ea985f6fdff0d166418a9e225122a9738666 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"560cc9cbeb912565bc6264f3a2e05b1a43e852396f08b5fa3ab6903b26a461d8","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"9bd091b6bca03f9058308213afcad70ef85c4865393a06afd1e9233ff9d2864d","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"e0b9e8e8e8c369304aa8c85b41c347fe25fa7bb2668b3865cafb3a7667b960c3","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"ad99fcbc68da63dfcfbf87ed8fc3c02294f3e11c760f038291b71836fe37b4b8","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"87f1612e5c7e91c0a20305b5b54f661ddae0a9b65b04d9ef888f0b4ba8406d1c","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"87d903bca20dc709976bb8a77f1bad8acca3f3d559939ffe9c27d40feffde1b5","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"84347fc19344c90f64963ca07c0b8501b2ee49f820a12bcc3ff629187a4f1860","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"f79b63694ca616fb3e101c1d25141f1f43fd0a8e8d5fb6f30085a5af3683d679","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"40bbe6beaee6037e6c51aaffc4b95b300f262012ddd0f1794ed1966800810fd9","created_at":"2026-08-07T07:26:24.033106+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-optimized200-routerstats-20260807b","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..8e2420f2fe86d449fc73757398bcf2796bc686e4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-optimized200-routerstats-20260807b/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"40bbe6beaee6037e6c51aaffc4b95b300f262012ddd0f1794ed1966800810fd9","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T07:26:24.354740+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..e1b130aee226b7fe575434be4ff3661fd827d857 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +cf22e55d2d22b685282a69d2fad23e4bd3d6cbb49629895d4479797a0210bdc3 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..160e53ef07b1878367ebeb19ba44ad62131705ef --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4671c80a23581ed03283c8bebbea128133b2c87fe6122b3527aa53a4797cfdcb +size 296513 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..92ef7a6b34b99eefdebcb79706fcfa645bfd0877 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cff92a1910717eda3911e5eaf02b12e342346708d3fb89aea17cdd3fcac3aaa +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ac5db366b255f66c7524f67a383135f71763c008 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"cf22e55d2d22b685282a69d2fad23e4bd3d6cbb49629895d4479797a0210bdc3","files":{"dcp/.metadata":"4671c80a23581ed03283c8bebbea128133b2c87fe6122b3527aa53a4797cfdcb","dcp/__0_0.distcp":"0cff92a1910717eda3911e5eaf02b12e342346708d3fb89aea17cdd3fcac3aaa","state.pt":"8e9261e90404527ad615c16c148c0eacddd1c09a6b2c927832f5d0641d366cc9"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"01e591d5addc388062e394a4b837eb33510f8eb622b6880d64826ae92ab96235","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..179c4eb3ce2a26b4285e90412198e70d434e558d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e9261e90404527ad615c16c148c0eacddd1c09a6b2c927832f5d0641d366cc9 +size 168499178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..f5c8d70c73de3884e1b579a6a32f2b809a2708f6 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +01e591d5addc388062e394a4b837eb33510f8eb622b6880d64826ae92ab96235 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..06140eef5996fc9f41f48126172c27cbb6fdf0ba --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1bdd6217163fb57626ef1ec46d4399b1e3e32b9be79e8fd0733e5816052ca73 +size 296507 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..92ef7a6b34b99eefdebcb79706fcfa645bfd0877 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cff92a1910717eda3911e5eaf02b12e342346708d3fb89aea17cdd3fcac3aaa +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a3a1bcac373efea7bbb48e67d5083544179d18d0 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"01e591d5addc388062e394a4b837eb33510f8eb622b6880d64826ae92ab96235","files":{"dcp/.metadata":"f1bdd6217163fb57626ef1ec46d4399b1e3e32b9be79e8fd0733e5816052ca73","dcp/__0_0.distcp":"0cff92a1910717eda3911e5eaf02b12e342346708d3fb89aea17cdd3fcac3aaa","state.pt":"9731e72cb84081f446df02081b5bf093cbb876757da40a84a44785462db77a55"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc8837bed9d425f367ba88304cf8b0a69da6d728 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9731e72cb84081f446df02081b5bf093cbb876757da40a84a44785462db77a55 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..2bf8ea4227bd7713108dd4e7ccb3e4e27c2a1b57 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-prefetch-nosync100-batch2000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_100_batch2000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"04a108c0524cc55f480e52e29151a9ac92cf7c0c258838c3c2bf56a71e88a882","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_100_batch2000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ffbe526cd6e72875d1408fe5462d87366098ca55 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"cd16b8ae2fb6874dd34e202996720015fdd040fc4578f0aa834b6dafadc2d74c","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"b4e906c5ebd1f0cb89191c71cafd28fb69503db84f9188c61e6a2e1fcb9e0803","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"6a5701fd60a96ce362b6e782ce39865b7b6aa6accd82898673b4b268572ebcad","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"fed5b2190a2e8c93183cee2886b231fdfb0a799a4af6872abfea4d7bc434df1c","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"ed97d243049a0d18f5efebea22c0cce3cb98489f449b03b3fda7b05c378622a9","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"abf8bb08a24e21324b7e10ad31f8d5dacdf93e95abc1f7a173786f2c3cb260cc","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"d526680d52f59a73149d243fd8615cdf5f65831391624afc18a0831113922357","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"eea5705f122676b591892482ba41fa956952cda2260f61e590d1ed070de4ed29","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"f452a78d458721190816f31c67875d8bb2323c6332e17635b2ca4baef7724e21","created_at":"2026-08-07T08:18:10.641554+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-prefetch-nosync100-batch2000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..5a9ddc763e3b72c3e98338e428d6eb75f2dfe720 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync100-batch2000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"f452a78d458721190816f31c67875d8bb2323c6332e17635b2ca4baef7724e21","reason":"BASE_ROUTER_GATE_FAILED","status":"FAILED_SCIENCE","updated_at":"2026-08-07T08:18:10.879801+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..e36dca592473b9419b2bf633293feafe75ea3ed7 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/COMMIT @@ -0,0 +1 @@ +2c9f80f9d15e4a174cc6287c808d7d3f33c64cf203a9370a2bb5a55df14b76e8 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9d541a8a769fcfe866f5af33b800ff8e2149ff88 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e38cbb7ca14c37f0a1ca9fc97702bf79c2e263b15e93ed527828e1762f882f +size 296513 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..15d350daaeaa2515b7eb64b212469d62cce231d4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"base-only-moe-final","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"2c9f80f9d15e4a174cc6287c808d7d3f33c64cf203a9370a2bb5a55df14b76e8","files":{"dcp/.metadata":"d3e38cbb7ca14c37f0a1ca9fc97702bf79c2e263b15e93ed527828e1762f882f","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"7d5d88161c0fc9c70621d64dd15a69dd8e35aea030e9817a1a616a05c2affcb9"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"d35f1d3ce8e4b9497a3d481bf245413d8c8e6f760279515394690b2c84af91b6","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..47dd327212f3e918cbdfb94bd7f29501b8df8269 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/base-only-moe-final/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5d88161c0fc9c70621d64dd15a69dd8e35aea030e9817a1a616a05c2affcb9 +size 168499178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..df1e21fa7adff088ecb51ae7ca4440fa7ae35f2e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +365cf7c30fcc6d5d51394999f7a2883e8accb7935d571545f24dcb2d3dcc79b0 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a908ec3deaa86fcb42e8f5e389b4cbb188383b59 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb6888206f452cfa4c1bbcc332ab13cba567186282237e5c64e9d04d93e10f46 +size 296507 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..db0272fe7dcfcf634712e6e74b8ac81df325070d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..f336728985f5c911035f25daa8521a82aebc5563 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"365cf7c30fcc6d5d51394999f7a2883e8accb7935d571545f24dcb2d3dcc79b0","files":{"dcp/.metadata":"eb6888206f452cfa4c1bbcc332ab13cba567186282237e5c64e9d04d93e10f46","dcp/__0_0.distcp":"263770b4a881c9dc4f4e94af16ebebe75304de722a36718ffd29744db9cb276f","state.pt":"a0c7e85a89e9a28662b914316df368716a1ec67c3b306c5d34ba339a9cf3c776"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":100000,"latent_tokens":102400000,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":100000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e50a93a0f76f05dc6935008abc5387240f1183d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0c7e85a89e9a28662b914316df368716a1ec67c3b306c5d34ba339a9cf3c776 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..e90dfb75b2462c6fd138f8317221a05a5d531478 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +d35f1d3ce8e4b9497a3d481bf245413d8c8e6f760279515394690b2c84af91b6 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c37ad68c7c549d1b88d3174f635872b55a95a5ca --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4edf76194ed51914034433f3303bf1dcd264016a69c53bbafe0ebdf3b383a731 +size 296507 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5a20546098dc607834467cd83cf280e87fde433 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086 +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..40e1886a19727e336452100bb2490c00b72f91c8 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"d35f1d3ce8e4b9497a3d481bf245413d8c8e6f760279515394690b2c84af91b6","files":{"dcp/.metadata":"4edf76194ed51914034433f3303bf1dcd264016a69c53bbafe0ebdf3b383a731","dcp/__0_0.distcp":"9a50107787b2a4d4da6452e8baf293fce5fc015cb3135b36363dc5e7310a3086","state.pt":"74d71aca216c003a0d2d87d0bf76be80460001009cd4d3367e7567333d1bb529"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"365cf7c30fcc6d5d51394999f7a2883e8accb7935d571545f24dcb2d3dcc79b0","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":200000,"latent_tokens":204800000,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":72000},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fe06a0af74539a356c312b6c9489ac4525dd1a5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74d71aca216c003a0d2d87d0bf76be80460001009cd4d3367e7567333d1bb529 +size 168491178 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..f3792824f95f17a9ce450f7c574642cb816f4d90 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-prefetch-nosync200-batch1000-20260807a","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"cc2cfec0ffa7713c69f9932a3ce1785eaaabf50d26405e91469829b8c15b757b","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/perf/plans/continuous_200_batch1000","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..5aa4f454ac22ddd8a804ec474497374babd44817 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"4296b14574734297a46bbc5eb749b493102b280b00ed6914d51864f0403453c0","name":"base_artifact_manifest","parent_hash":null,"path":"base/base_artifact_manifest.json","schema_version":"1.0.0"},{"hash":"c0a0e2a979874300cb03bab3adf3ef4094aa23b2dc283eb8b479d7e9fd00bab0","name":"base_loss_history","parent_hash":null,"path":"base/base_loss_history.json","schema_version":"1.0.0"},{"hash":"7c8130438e4eb536d6619412d01bc1cb69511bf5ae8730e78e85f7c1e9326989","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"5183bb961b9ce47ba50b5d307b5cd783995a12c9e1ffd05234009a54d411f272","name":"base_training_config","parent_hash":null,"path":"base/base_training_config.json","schema_version":"1.0.0"},{"hash":"138b913011dc9130922413786a83b33dfd2b95b3173079b268ea5e67db6a856d","name":"base_training_log","parent_hash":null,"path":"base/base_training_log.json","schema_version":"1.0.0"},{"hash":"e2e1a78d0ca4a6a71be1ae2fff137b892a5073fc7719968be63365cb1afe24e8","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/base-only-moe-final/manifest.json","schema_version":"1.0.0"},{"hash":"9af03c5412508690bf70855b87f5bacaf2d0a918d3d3d4f3fb6fa0885d4791ed","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"eea5705f122676b591892482ba41fa956952cda2260f61e590d1ed070de4ed29","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"8f5fb474761bcb7da3a3b298c615c46ef452c7d308040ccea7280be489b28b93","created_at":"2026-08-07T08:15:42.104691+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-prefetch-nosync200-batch1000-20260807a","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..bf173cd8b3d6a7a3141b1f34e342ed4a4e04c667 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-prefetch-nosync200-batch1000-20260807a/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"8f5fb474761bcb7da3a3b298c615c46ef452c7d308040ccea7280be489b28b93","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T08:15:42.385919+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..1f224e4209cb9b79064ee9a3dd6536ae56ca662d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +be331df74e8d7f429249675043bfff4ac9fe9809ba47898d01c15064bbf0d21b diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..36d69151ba3ad226588b5362f0c86ec02ad2f9c0 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff64eb0f823deed16408f91b28509169284bc93fe8532c56a64738dbe7f635ce +size 296485 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bd60b742a7881893f2f14c5901eef290610102a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c +size 169113792 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..509cf8403c1b1dc222dce0dcbad90cecb6dbd525 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"be331df74e8d7f429249675043bfff4ac9fe9809ba47898d01c15064bbf0d21b","files":{"dcp/.metadata":"ff64eb0f823deed16408f91b28509169284bc93fe8532c56a64738dbe7f635ce","dcp/__0_0.distcp":"4108bb4b1c4f313dad06306f76e8bbda01dbabf8f28904d2a722c0b1132fbf1c","state.pt":"e9ffb6b8263cfed52ba92f30b4b3a62a2b5c1e42f145b3a87335f04a70015e22"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.002,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"25a79bd702c41ddc230e9da49950848570ded039885929998395fc7a2838924d","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"dce85925176d549a1b865e46dd99a64e3eddc32cf88db75678df8a397b29ef3b","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..c00186baa4fe27afb2dc0777998b31386e866bc4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ffb6b8263cfed52ba92f30b4b3a62a2b5c1e42f145b3a87335f04a70015e22 +size 168500266 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..d1f07bfdad886a1e3357abac0c743dafc409bc00 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-resumed200-exact","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/resume_from_continuous_step100","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100"],"input_hashes":{"base_train_plan":"c2e9e0461166fb9bfa5c98de7c90608a4071888a9a3482bf6a5cdcff2122a9a6","checkpoint":"dce85925176d549a1b865e46dd99a64e3eddc32cf88db75678df8a397b29ef3b","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/plans/resume_from_continuous_step100","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c8730098d090969ccaed7cfdcc18d845cfcb0c51 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"0a2b38f383598fe43c2079ca3da6df49722a2a4836874798c87099356841f663","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"860bddb75f89deecf45ad29cb435b1302e092c114f289af993024bc55a184362","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00000200/manifest.json","schema_version":"1.0.0"},{"hash":"1fd1183ee1d687526948fb0e3368c76a1e8b08ca999327c027426ffa4348ab02","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"5119c6e1144d2a3b4e731efbccbeec9243d1016ebc48e2d245d90a430f4cbd43","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"6c1f1cbf03f8cae3d068caf1fad0f4246b5ed4774aefe80abfe6ddee9b2244b6","content_hash":"bc4e1d0d030b32ac7068c0379107b8cae2a439d73595b5eb9fbf0906cf83bbbd","created_at":"2026-08-07T03:25:45.943145+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"dce85925176d549a1b865e46dd99a64e3eddc32cf88db75678df8a397b29ef3b","run_id":"dev-e15-base-resumed200-exact","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..cb1f973fc634a7664eb57a0c3b611e41e432bd69 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/formal-eta002/runs/dev-e15-base-resumed200-exact/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"bc4e1d0d030b32ac7068c0379107b8cae2a439d73595b5eb9fbf0906cf83bbbd","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T03:25:46.336947+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..840e46c43067adf39fed42d55ccdecddb5e5153f --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3ccbd88da3baec57e8b3f35c4d561774b3f03673 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b60f5d3c91dbdb75a67a2e97686db09651c22fd6fbf6629cc1b5bb4b1cc9e57 +size 287339 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f49c62bbbb0a88deaadb3222301ffedcf40e89d9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d281b178e5d3048942704619cc8ddd3e4ba2e48ab4c80f78c35138a58e04e98 +size 169076032 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..f165f4854b2686ffc4da0f324e433922f0d50e10 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11","files":{"dcp/.metadata":"1b60f5d3c91dbdb75a67a2e97686db09651c22fd6fbf6629cc1b5bb4b1cc9e57","dcp/__0_0.distcp":"6d281b178e5d3048942704619cc8ddd3e4ba2e48ab4c80f78c35138a58e04e98","state.pt":"46ba44341afa7ea6a6fb15913c4ab193b6c035a8423fcb800c9c3844644115d3"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"f5d26646d9a0aaa844a9c8ebb652298361867b5e4ca9d10ef3dcda3b1df4ed41","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25600,"latent_tokens":26214400,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":25600},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..015fb693b0bfdbffc8a566b897a8b0834a63a550 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ba44341afa7ea6a6fb15913c4ab193b6c035a8423fcb800c9c3844644115d3 +size 168481834 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..911088aebad84b4fe3ca49d470b6dce5fabb8d8a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +b5f0308c708bdafcaeeee1848224d2c7e7dcb18b6ced290b62953148ad43e643 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9d3ebd9e8791cec9cea6683ed8a859dd0c80be28 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2fcb4c8869c515fc71044a0cbe515e062778fc0953b6136f1aabbf542faa0ad +size 287339 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..69b1bb4fba09b02ef9d5bb5dddf736de08b1b4bb --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b6900151d55084d6b564666e3d1018a9487a28bd384980c24b336ba7d458c4 +size 169076032 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..95475c7f449a8846ba1c785772d5cf47aad62fcf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"b5f0308c708bdafcaeeee1848224d2c7e7dcb18b6ced290b62953148ad43e643","files":{"dcp/.metadata":"a2fcb4c8869c515fc71044a0cbe515e062778fc0953b6136f1aabbf542faa0ad","dcp/__0_0.distcp":"07b6900151d55084d6b564666e3d1018a9487a28bd384980c24b336ba7d458c4","state.pt":"9ce626b35a019f53352b510b6be881435c852a059cf3ffa7f5d1d73a5b036a16"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"f5d26646d9a0aaa844a9c8ebb652298361867b5e4ca9d10ef3dcda3b1df4ed41","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..3edea71dbeff8542e0d467daced567124378eddf --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ce626b35a019f53352b510b6be881435c852a059cf3ffa7f5d1d73a5b036a16 +size 168481834 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..d3c43c3f16b1656dc7e8817050229cf86fa26134 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-continuous200","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/continuous_200","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"c4e965751dbeaac9d651a197ec8f730d6cc34d1fa03e7426b2cdf62ac6fc3ea0","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/continuous_200","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6f904fd80a73e2042846849bff75b8276f0b545b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"3766823a45a075dd67164bc559194d9e6c83a8e50b7b556c0137a8a76726d43d","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"32ef1b199eb16785b123a7c9a359038a506e2461b9b64b31c55c6286a0d4eb12","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00000200/manifest.json","schema_version":"1.0.0"},{"hash":"407b6e878894705621321d2fc5afd9400c074cd683c5d8a6840c737bcd9a132b","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"42db2743a72d932bdb530ff4257a9adf1b9d22f331f8fa17b3fdde636b98bb6a","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","content_hash":"ffac8ae4ec9a718360aafce19c3ed59bbd07936c4ca6a89768b6f88e7ab5430b","created_at":"2026-08-07T02:15:31.164096+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-continuous200","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..427a5d9142f332a0f912edd9b378d6b6d78efe5c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"ffac8ae4ec9a718360aafce19c3ed59bbd07936c4ca6a89768b6f88e7ab5430b","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T02:15:31.602073+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..e1affeab7cd6b8d2f74bf81d8063df5987ffbd3a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/COMMIT @@ -0,0 +1 @@ +84cdc8a288891654ec25c5bcced7384b7a62b91e007dbcb95065f8fb9664ff44 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4aa9ea6a30d7ffac07a6027893d562d825bfc067 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de1b223060712d0635e8d8cde5671248b614de3bb136242351afc55991c04a29 +size 287335 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f49c62bbbb0a88deaadb3222301ffedcf40e89d9 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d281b178e5d3048942704619cc8ddd3e4ba2e48ab4c80f78c35138a58e04e98 +size 169076032 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6be47ff211cdcecda5ede2c482d6880a06bbd46a --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000100","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"84cdc8a288891654ec25c5bcced7384b7a62b91e007dbcb95065f8fb9664ff44","files":{"dcp/.metadata":"de1b223060712d0635e8d8cde5671248b614de3bb136242351afc55991c04a29","dcp/__0_0.distcp":"6d281b178e5d3048942704619cc8ddd3e4ba2e48ab4c80f78c35138a58e04e98","state.pt":"0805d792d217e1fcd49e80bfd635e6d587da0f37c3662f29a5fb676adcdd611b"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"f5d26646d9a0aaa844a9c8ebb652298361867b5e4ca9d10ef3dcda3b1df4ed41","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25600,"latent_tokens":26214400,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":100,"sampler_cursor":25600},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..e899feb85ca28018cd2717349c816241174091b4 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/base/checkpoints/step-00000100/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0805d792d217e1fcd49e80bfd635e6d587da0f37c3662f29a5fb676adcdd611b +size 168481834 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..c57899db928f9044c1aa9aac27ef20f1be3f3bb2 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-resume100","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/resume_initial_100","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"89c21354c19d49b0b6d09d4baf1f76833a9232245725632a9b17f97449b98009","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/resume_initial_100","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..62d6137e96026ddff3dfe2931b415060b853e117 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"aabcad9e6612eac5e58b4d10f325731f9de328c885d7755f397140ccacd9f1fa","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"7a6e1f1364ac65c457b159313a70e4cf6a5a728daa5b5aea28fec38a8d624a62","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00000100/manifest.json","schema_version":"1.0.0"},{"hash":"b98fe461e7748006f27873b672e51578f3e3377192296dc3a824f0bd9a5525f8","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"42db2743a72d932bdb530ff4257a9adf1b9d22f331f8fa17b3fdde636b98bb6a","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","content_hash":"a05e9f78c08664f7da7c347954fe78c3ca9ba580f3fdb6da398214c3e804412c","created_at":"2026-08-07T02:16:30.788332+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-resume100","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..e4d50804f2da2738fe9c532dcf0756e1e3451135 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resume100/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"a05e9f78c08664f7da7c347954fe78c3ca9ba580f3fdb6da398214c3e804412c","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T02:16:31.167070+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..7a8c17a78207d1efca7a508a7cddf380beb1546c --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/COMMIT @@ -0,0 +1 @@ +af3d6f7ab4ee8bfc4616996f0cd50b56b588a9c6312f78fef0a32bffb12ff366 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1aa3c24493b60d6ae36ee1eaab10867af85a866e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19ba3810f5b9ccb9decaf381c6fe556f70a8bdcfbe06df7e4d4edf050e35e04 +size 287342 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..69b1bb4fba09b02ef9d5bb5dddf736de08b1b4bb --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b6900151d55084d6b564666e3d1018a9487a28bd384980c24b336ba7d458c4 +size 169076032 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..045f09f0cca7bedfa950b5505414068f99c466b7 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000200","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"af3d6f7ab4ee8bfc4616996f0cd50b56b588a9c6312f78fef0a32bffb12ff366","files":{"dcp/.metadata":"c19ba3810f5b9ccb9decaf381c6fe556f70a8bdcfbe06df7e4d4edf050e35e04","dcp/__0_0.distcp":"07b6900151d55084d6b564666e3d1018a9487a28bd384980c24b336ba7d458c4","state.pt":"032b976713de00f3f6f45f929450c4977f02b1d37fcbc4769ee83b1e774a694b"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"f5d26646d9a0aaa844a9c8ebb652298361867b5e4ca9d10ef3dcda3b1df4ed41","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11","role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":51200,"latent_tokens":52428800,"microstep":0,"optimizer_step":200,"phase":"base","phase_step":200,"sampler_cursor":51200},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..42d1615f748e732d4c64c5d00133ba6c8c5e1c9b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/base/checkpoints/step-00000200/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:032b976713de00f3f6f45f929450c4977f02b1d37fcbc4769ee83b1e774a694b +size 168491242 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..5214386da4809408c652889feb2673cc031fb964 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-resumed200-exact","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/resume_from_continuous_step100","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100"],"input_hashes":{"base_train_plan":"cd983ef95dab08e395d5eb6f81bbc0918a5dc63149d8a85a7ef8e814eb5bf5a8","checkpoint":"73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/resume_from_continuous_step100","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-continuous200/base/checkpoints/step-00000100","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..f211ade0421978f730dc103a999a3bc63a3d9054 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"99ee97e752ca8bd458a8a7c4f0e1bcf8f6a8424b806fea2639978bf945027c26","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"410de8d46a0b328e181802b81bbf7fa9980b7b2df118f446ccaf501f14b5ba73","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00000200/manifest.json","schema_version":"1.0.0"},{"hash":"bcfad10301e44787bdeb813dee9f61fce00cc73ddec37824d45a52acefba31ce","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"42db2743a72d932bdb530ff4257a9adf1b9d22f331f8fa17b3fdde636b98bb6a","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","content_hash":"c47eadba0506057786b0f32b066878b1928869fb03a4c458ab40e9cbd1e17cc5","created_at":"2026-08-07T02:23:35.137489+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"73ccb8242acd8f687a0ac61fe763a4f6d886701af97e1138d8f6c10668396e11","run_id":"dev-e15-base-resumed200-exact","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..7e32fd4672fd7112b9c1e8cdedeb0608f079862e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-resumed200-exact/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"c47eadba0506057786b0f32b066878b1928869fb03a4c458ab40e9cbd1e17cc5","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T02:23:35.712990+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/COMMIT b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..1eb58454313f208ec58ad3c98600e3490238c1f5 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/COMMIT @@ -0,0 +1 @@ +a3c6c24df790ec1a289105fa4343df36b973a1dc82ea7c3342c999dc49d74990 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/.metadata b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0cd5cdabb3029a62605cf3ee4ff00a2de2e4a58b --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7191b00c2a7d2b68f8263f507280fb13021346554329793687e5a3b5b351052 +size 287338 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/__0_0.distcp b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8d4da0154bf3287df5863b3838789851dc85cc51 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84457703c54986a4650c7f29c4e2901ff509050ce27ec7072e3c903e4ea5e941 +size 169076032 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..aad11cc793780045f8061c265d0d3192a2c27b7d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"step-00000010","component_names":["ema","engine","model","optimizer","sampler","scheduler"],"content_hash":"a3c6c24df790ec1a289105fa4343df36b973a1dc82ea7c3342c999dc49d74990","files":{"dcp/.metadata":"e7191b00c2a7d2b68f8263f507280fb13021346554329793687e5a3b5b351052","dcp/__0_0.distcp":"84457703c54986a4650c7f29c4e2901ff509050ce27ec7072e3c903e4ea5e941","state.pt":"3135cd44b9e25e3831c89c9af23e3c165c47388e39440782ddc56bc439b08717"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_beta0":null,"pair_router_feature_norm_epsilon":1e-05,"pair_router_has_null":true,"pair_router_null_logit_init":2.0,"pair_router_probability_epsilon":1e-08,"pair_router_temperature":1.0,"pair_router_temperature_end":0.1,"pair_router_temperature_schedule":"linear","rank":16,"separable_rank":12,"slot_capacity":2},"model":{"depth":8,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":320,"hidden_size":320,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7],"num_classes":100,"num_heads":5,"out_channels":4,"patch_size":4,"variant":"mini"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"f5d26646d9a0aaa844a9c8ebb652298361867b5e4ca9d10ef3dcda3b1df4ed41","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":null,"role_spec_hash":"23e60b4260edafc09b8808cf330d989a08de08579276dda5dc7db777b7c5c315","schema_version":"1.0.0","shadow_commit_hash":null,"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":2560,"latent_tokens":2621440,"microstep":0,"optimizer_step":10,"phase":"base","phase_step":10,"sampler_cursor":2560},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/state.pt b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..b319e1c6d5efe6b907f672cfb8d13f63a251ffc6 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/base/checkpoints/step-00000010/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3135cd44b9e25e3831c89c9af23e3c165c47388e39440782ddc56bc439b08717 +size 168481834 diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/launch.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..3983ceb78bb39434eb2f56528ffde161e190727d --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:base_stage_service","argv":["--run-id","dev-e15-base-throughput10","--runs-root","/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs","--config-dir","/workspace/gsh/tmp/Schatten_MoE-continuation/configs","--config-name","e15","--input","base_train_plan=/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/throughput_10","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"],"input_hashes":{"base_train_plan":"633838243af442e6dc35e9092300c0187ad23f273c730fc6102bc976adbea89f","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"base_train_plan":"/workspace/gsh/tmp/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/plans/throughput_10","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":[],"stage":"train_base"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_manifest.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8ff467798c4acc34e72707e0d3a22f9eba95fe6e --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"5c21a8517734e1abf7c2fe2c4cdc3456cda517670424e5d2101bb719757fb2d5","name":"base_metrics","parent_hash":null,"path":"base/base_metrics.json","schema_version":"1.0.0"},{"hash":"5e30a3ea46cca441b478b157a7b213720b1f9e519e2b7c5515c8e29886982761","name":"checkpoint","parent_hash":null,"path":"base/checkpoints/step-00000010/manifest.json","schema_version":"1.0.0"},{"hash":"c497b7719ee4203e5225352490e53b78ffd47eec649f96a67d0f105af98a5cf5","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"42db2743a72d932bdb530ff4257a9adf1b9d22f331f8fa17b3fdde636b98bb6a","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"bb8397daf2a11c6310b8707e7ad80a4f11b770ad251764e2f04cad5a9cc48c7c","content_hash":"5b90052267d07763e701e2ff4b11409028c3aec44e77407c14b608a2d5211cd6","created_at":"2026-08-07T02:13:35.870027+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":null,"run_id":"dev-e15-base-throughput10","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1731,"augmentation":1706,"candidate_audit":1741,"capture_sampling":1711,"cfg":1705,"data_order":1701,"diffusion":1703,"generation":1751,"global_seed":1701,"posterior":1702,"shadow_init":1721,"timestep":1704},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_base","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_status.json b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..729bb7b6449854c8d4012ac5ff450f04c3eee694 --- /dev/null +++ b/artifacts/Schatten_MoE-e15-experiments/E15-mini-imagenet100-base1701/runs/dev-e15-base-throughput10/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"5b90052267d07763e701e2ff4b11409028c3aec44e77407c14b608a2d5211cd6","reason":null,"status":"COMMITTED","updated_at":"2026-08-07T02:13:37.020991+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/launch.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..da7903c313f677e94fdc9215f5b0b8b3ab48d693 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:proxy_stage_service","argv":["--run-id","E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e","--runs-root","/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","--config-dir","/workspace/gsh/tmp/codex_tel_work/Schatten_MoE/Schatten_MoE-main/configs","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","shortlist=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","--input","candidate_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","--input","proxy_schedule=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-046-candidate-6a69f7122572552c3aa4/manifest.json","experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"input_hashes":{"candidate_manifest":"d563804591bfa5acb8792eca7150a6388e5958883eac3f6a846b33fcdefcd37a","checkpoint":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","proxy_schedule":"97cc77146d518d05398534b318bc625528069418c52f065d1e544a96b794cbcb","shortlist":"95777acb53d562c552bc5825ddd06bc26919172c065ff5f93d3bbb8467f5d743","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"candidate_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","proxy_schedule":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-046-candidate-6a69f7122572552c3aa4/manifest.json","shortlist":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":["experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"stage":"train_proxy"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..c842dbf75e021b9de994e819dbfd786d07bae9d2 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT @@ -0,0 +1 @@ +6f3be76c9114f3e2c9d8ca60b2d7a314f6be8702bd16ab5117eee2365f67e712 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6e5efe94c452c3b787d89949ef3906fe88985b35 Binary files /dev/null and b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata differ diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2b6e73b73e7bffc3a0e31beaa42a6940076525cb --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f95d9897c88bce53bbb9c3ff78f6a98131f9b4bfe1433f81886156c9d4ea7403 +size 254224 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..9631d31e35270b824f5a4588c1d4350809279863 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"shadow-bank-proxy","component_names":["shadow_bank"],"content_hash":"6f3be76c9114f3e2c9d8ca60b2d7a314f6be8702bd16ab5117eee2365f67e712","files":{"dcp/.metadata":"74706c8316090c761e29b2886c80c88dec05ca8ac5cd0f0005022fdb7dd23356","dcp/__0_0.distcp":"f95d9897c88bce53bbb9c3ff78f6a98131f9b4bfe1433f81886156c9d4ea7403","state.pt":"10e22ea25094c856149d75b0177255bd972fc8383b0129aa471071fefddf359d"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"a26691553c149c42fcc9ee89dcd098e6ec49d11ee75127e96c14ddfda9344dc2","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_has_null":true,"pair_router_temperature":1.0,"rank":16,"slot_capacity":1},"model":{"depth":12,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":384,"hidden_size":384,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7,8,9,10,11],"num_classes":100,"num_heads":6,"out_channels":4,"patch_size":4,"variant":"tiny"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"a424098031c3112246695b58cb832e27fb28305df596984aabfd6650aef19698","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","role_spec_hash":"bc27fc8428b29ff36a023165e6dd27446e64d1ae7a4a485a7deb6c5444b7676c","schema_version":"1.0.0","shadow_commit_hash":"b25b72eb4a9a13fc87959cae3358188a1d9d5d95a1f6a6b0c70c3b3ce476cda8","split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25521,"latent_tokens":0,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":0,"sampler_cursor":null},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..07d57e74299d5263ec679db44d92cd2e04cf075c --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e22ea25094c856149d75b0177255bd972fc8383b0129aa471071fefddf359d +size 3157378 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..999c902a588d7cc37f165d49d165f4a55d6a7819 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"b25640c55f6f3e067090bb305e9f22586a451713b531f1ee22fc1ef11f1b0800","name":"proxy_cache_manifest","parent_hash":null,"path":"proxy/cache/manifest.json","schema_version":"1.0.0"},{"hash":"bfe29da70c0a616b58d267019147b7e80a08c458b09ce1243a4107c56f5b2775","name":"proxy_evidence","parent_hash":null,"path":"proxy/evidence/manifest.json","schema_version":"1.0.0"},{"hash":"54b2350533f621d828f61af6b67c217086c3826baac49927d162682626e4a421","name":"proxy_windows","parent_hash":null,"path":"proxy/proxy_windows.json","schema_version":"1.0.0"},{"hash":"4cfe67eec42e48fd711d33fc85569bb17685ceb4604947d68662de7d353d607b","name":"shadow_bank_checkpoint","parent_hash":null,"path":"proxy/checkpoints/shadow-bank-proxy/manifest.json","schema_version":"1.0.0"},{"hash":"7966e3fa91830d166f3100adab8a36de086d37e3442ba7b9613fa817b13daa6d","name":"shadow_bank_receipt","parent_hash":null,"path":"proxy/bank_receipt/manifest.json","schema_version":"1.0.0"},{"hash":"c93e627bc3644f8124bbddfc29c02f45bf458e60aef0050e070e9c6b6e21e809","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"893274bcdbe1420557cf2695ecb0c9806c799efb46bf1b7ce56699a698f636b2","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"4c3ad4fbd6c0b21c2c19745e27516f0e4a557b16c7666aaa109a2b525da5b8be","content_hash":"6e653c64265fc0be400fccd0e09c7e655d50f576f987902c61ef547bd720ff81","created_at":"2026-08-03T22:12:11.104013+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","run_id":"E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1010,"augmentation":1006,"candidate_audit":6006,"capture_sampling":1007,"cfg":1005,"data_order":1001,"diffusion":1003,"generation":1011,"global_seed":1701,"posterior":1002,"shadow_init":1009,"timestep":1004},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_proxy","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_status.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..aa8b12254134db322b115bfc41aa4a08214e3196 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p046-3abed5e/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"6e653c64265fc0be400fccd0e09c7e655d50f576f987902c61ef547bd720ff81","reason":null,"status":"COMMITTED","updated_at":"2026-08-03T22:12:12.114460+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/launch.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..dd963b733ff2026d9e4917413ce0fbaab717ca1a --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:proxy_stage_service","argv":["--run-id","E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e","--runs-root","/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","--config-dir","/workspace/gsh/tmp/codex_tel_work/Schatten_MoE/Schatten_MoE-main/configs","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","shortlist=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","--input","candidate_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","--input","proxy_schedule=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-061-candidate-ae4026855da8143f4819/manifest.json","experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"input_hashes":{"candidate_manifest":"d563804591bfa5acb8792eca7150a6388e5958883eac3f6a846b33fcdefcd37a","checkpoint":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","proxy_schedule":"91158a1487a881d1dedb4af13e15bb3bfa18428658a597248aed42fb7c7970fc","shortlist":"95777acb53d562c552bc5825ddd06bc26919172c065ff5f93d3bbb8467f5d743","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"candidate_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","proxy_schedule":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-061-candidate-ae4026855da8143f4819/manifest.json","shortlist":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":["experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"stage":"train_proxy"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..f4d6302ba44f5d08acf2385eb9664ef64158a800 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT @@ -0,0 +1 @@ +716c8235eca1cacca86550f9faaaa8553497cd2f8ca4739e67d1ba10d05ed26f diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f4387646dc0d655a406609da4c21dcb418191003 Binary files /dev/null and b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata differ diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..837a0768132c7fa33f2c3391d2fa6d638cc9b5b2 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a275cad2837460c5ea9d90b94f455ca6be11854acf0fe46ed3f59b1e8273ab3 +size 254224 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..fa9b9ab4f01a85562c22251aa63968c7611e620e --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"shadow-bank-proxy","component_names":["shadow_bank"],"content_hash":"716c8235eca1cacca86550f9faaaa8553497cd2f8ca4739e67d1ba10d05ed26f","files":{"dcp/.metadata":"1dd2494cd5afcce5163b0df79a75011991b69a4ed996ee508dd164b4ec9746ae","dcp/__0_0.distcp":"4a275cad2837460c5ea9d90b94f455ca6be11854acf0fe46ed3f59b1e8273ab3","state.pt":"0fef22b10a35d134d532c5689e0344ba3f3a968b28d9e7d7504175d5c652cc2a"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"a26691553c149c42fcc9ee89dcd098e6ec49d11ee75127e96c14ddfda9344dc2","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_has_null":true,"pair_router_temperature":1.0,"rank":16,"slot_capacity":1},"model":{"depth":12,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":384,"hidden_size":384,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7,8,9,10,11],"num_classes":100,"num_heads":6,"out_channels":4,"patch_size":4,"variant":"tiny"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"a424098031c3112246695b58cb832e27fb28305df596984aabfd6650aef19698","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","role_spec_hash":"bc27fc8428b29ff36a023165e6dd27446e64d1ae7a4a485a7deb6c5444b7676c","schema_version":"1.0.0","shadow_commit_hash":"0270e27568f307485351d1a9767bc21146a98d5dd9c7bb1880f78ba6e9af7722","split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25522,"latent_tokens":0,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":0,"sampler_cursor":null},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..8598a2e5521ae9deec12d338ad38639374f11968 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fef22b10a35d134d532c5689e0344ba3f3a968b28d9e7d7504175d5c652cc2a +size 3195650 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..75e8849cc58d19e8df2fdb39610c5a7a6b25b059 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"3a1d69babdb04e9da633d8b19c2f48bb9c299eb2aebc56fd8c55e2ef31186361","name":"proxy_cache_manifest","parent_hash":null,"path":"proxy/cache/manifest.json","schema_version":"1.0.0"},{"hash":"807008600fed3ba787589a7a93c80c7918a272f7cfbdcf888a958a071e98d6af","name":"proxy_evidence","parent_hash":null,"path":"proxy/evidence/manifest.json","schema_version":"1.0.0"},{"hash":"edbe2b1f2d891297f6f519df74b232f4fdd5c066cf774eb954cdc75d22826487","name":"proxy_windows","parent_hash":null,"path":"proxy/proxy_windows.json","schema_version":"1.0.0"},{"hash":"4ff3e9745ae8f882778d700ba08154d25be3c2e5a200a68ed745551afb492a06","name":"shadow_bank_checkpoint","parent_hash":null,"path":"proxy/checkpoints/shadow-bank-proxy/manifest.json","schema_version":"1.0.0"},{"hash":"60079b36b075f3fbe232892fcb0496c38fb7235e87ea8ce4923ad737afe95666","name":"shadow_bank_receipt","parent_hash":null,"path":"proxy/bank_receipt/manifest.json","schema_version":"1.0.0"},{"hash":"c2f057fd9aad0c5094cacd4beb84c1bdba08ae4e13f24cb545af6a8482b7c64c","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"893274bcdbe1420557cf2695ecb0c9806c799efb46bf1b7ce56699a698f636b2","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"4c3ad4fbd6c0b21c2c19745e27516f0e4a557b16c7666aaa109a2b525da5b8be","content_hash":"8bb7d859313af470063705135357872b8841b63fa7fe59d07ed4b5092c8a8fe7","created_at":"2026-08-03T16:52:35.075944+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","run_id":"E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1010,"augmentation":1006,"candidate_audit":6006,"capture_sampling":1007,"cfg":1005,"data_order":1001,"diffusion":1003,"generation":1011,"global_seed":1701,"posterior":1002,"shadow_init":1009,"timestep":1004},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_proxy","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_status.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..ab59b0475f7c7600522f5c08a7c2d70ae5210c1f --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p061-3abed5e/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"8bb7d859313af470063705135357872b8841b63fa7fe59d07ed4b5092c8a8fe7","reason":null,"status":"COMMITTED","updated_at":"2026-08-03T16:52:36.188018+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/launch.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..c52456415a6a19c477dd81c2f9c1a7fc332361fa --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:proxy_stage_service","argv":["--run-id","E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e","--runs-root","/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","--config-dir","/workspace/gsh/tmp/codex_tel_work/Schatten_MoE/Schatten_MoE-main/configs","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","shortlist=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","--input","candidate_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","--input","proxy_schedule=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-063-reverse-bd8696ea66c9bb774143/manifest.json","experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"input_hashes":{"candidate_manifest":"d563804591bfa5acb8792eca7150a6388e5958883eac3f6a846b33fcdefcd37a","checkpoint":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","proxy_schedule":"0beadda3c18276c41ac47860d550662446029457c7e38d516df05e9748b7bfde","shortlist":"95777acb53d562c552bc5825ddd06bc26919172c065ff5f93d3bbb8467f5d743","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"candidate_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","proxy_schedule":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-063-reverse-bd8696ea66c9bb774143/manifest.json","shortlist":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":["experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"stage":"train_proxy"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..f053b51733b86476322f9217d3971665f41ae17c --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT @@ -0,0 +1 @@ +1b387aad41ca43e509eb3a7c74e008320e7afad36b45ee3c84342c6a4fc4d488 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..271d3faa731bb4fcc42a4b450ab268dd3826ac74 Binary files /dev/null and b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata differ diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6504fd8a96024ecb27cfd97e5d13a8187c86d01b --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae1f11ca298dcaaa3493d81a04cb74e55ef76d7d9d7bbdda413e9c3191ba6e9e +size 254224 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..520977f6492312b810f9137ac2e64232327dda56 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"shadow-bank-proxy","component_names":["shadow_bank"],"content_hash":"1b387aad41ca43e509eb3a7c74e008320e7afad36b45ee3c84342c6a4fc4d488","files":{"dcp/.metadata":"c9f122f47789a830dac98ba703615db3f4ce64dd54af176601579e3c5853863d","dcp/__0_0.distcp":"ae1f11ca298dcaaa3493d81a04cb74e55ef76d7d9d7bbdda413e9c3191ba6e9e","state.pt":"0b511b8fe165bc7a74db66be4e0bd0689a43198e329ac255d0bb378ac0318513"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"a26691553c149c42fcc9ee89dcd098e6ec49d11ee75127e96c14ddfda9344dc2","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_has_null":true,"pair_router_temperature":1.0,"rank":16,"slot_capacity":1},"model":{"depth":12,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":384,"hidden_size":384,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7,8,9,10,11],"num_classes":100,"num_heads":6,"out_channels":4,"patch_size":4,"variant":"tiny"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"a424098031c3112246695b58cb832e27fb28305df596984aabfd6650aef19698","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","role_spec_hash":"bc27fc8428b29ff36a023165e6dd27446e64d1ae7a4a485a7deb6c5444b7676c","schema_version":"1.0.0","shadow_commit_hash":"6ce86514f11e20d3598187acd87c9f294a17da592e3238c67a18b5c702f2bd39","split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25539,"latent_tokens":0,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":0,"sampler_cursor":null},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a2bca3f74546657f3e7ca6564e7f72d723f9431 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b511b8fe165bc7a74db66be4e0bd0689a43198e329ac255d0bb378ac0318513 +size 3148482 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d04899ab1308339c443eb376be8c9a140b97cfe9 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"6ba48b5240088a2cec7d472187dd039648b3f74f2062ac1a1f0ef04a2753cb60","name":"proxy_cache_manifest","parent_hash":null,"path":"proxy/cache/manifest.json","schema_version":"1.0.0"},{"hash":"bf961e9609243611dad0247e9464a640aac56910a07bd1b1b5ebee6a30559d9d","name":"proxy_evidence","parent_hash":null,"path":"proxy/evidence/manifest.json","schema_version":"1.0.0"},{"hash":"408ea632456fdf59d73924cd646f9d9a7059f570ecad60b7aa02137672e459ec","name":"proxy_windows","parent_hash":null,"path":"proxy/proxy_windows.json","schema_version":"1.0.0"},{"hash":"5ba0422e435bf08201a1c5a2eea6611c651d02943899a5c3333e15a627dd45ed","name":"shadow_bank_checkpoint","parent_hash":null,"path":"proxy/checkpoints/shadow-bank-proxy/manifest.json","schema_version":"1.0.0"},{"hash":"fc54a57d19f3c8800d1628e0767ceca14924a5f5b744971f15d1a7bb879d110a","name":"shadow_bank_receipt","parent_hash":null,"path":"proxy/bank_receipt/manifest.json","schema_version":"1.0.0"},{"hash":"17b2df1bb2865a2845c161462a9d17eb3e4d8c647a3688b29e9ecfc032fb5ecb","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"893274bcdbe1420557cf2695ecb0c9806c799efb46bf1b7ce56699a698f636b2","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"4c3ad4fbd6c0b21c2c19745e27516f0e4a557b16c7666aaa109a2b525da5b8be","content_hash":"d7e0d2e92b532fca3b96f53a9e04f4c1ccc3b1a92c933abd70b5dcaa49051ad8","created_at":"2026-08-04T00:57:04.495657+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","run_id":"E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1010,"augmentation":1006,"candidate_audit":6006,"capture_sampling":1007,"cfg":1005,"data_order":1001,"diffusion":1003,"generation":1011,"global_seed":1701,"posterior":1002,"shadow_init":1009,"timestep":1004},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_proxy","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_status.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..03da02ad716297da8892f0be2c5148c3d6a5d107 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p063-3abed5e/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"d7e0d2e92b532fca3b96f53a9e04f4c1ccc3b1a92c933abd70b5dcaa49051ad8","reason":null,"status":"COMMITTED","updated_at":"2026-08-04T00:57:05.262994+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/launch.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..92de9121364e65d1e4cbb92d3b45316893c6507c --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:proxy_stage_service","argv":["--run-id","E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e","--runs-root","/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","--config-dir","/workspace/gsh/tmp/codex_tel_work/Schatten_MoE/Schatten_MoE-main/configs","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","shortlist=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","--input","candidate_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","--input","proxy_schedule=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-071-candidate-bc3aed01c13dd69fa821/manifest.json","experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"input_hashes":{"candidate_manifest":"d563804591bfa5acb8792eca7150a6388e5958883eac3f6a846b33fcdefcd37a","checkpoint":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","proxy_schedule":"4c6316289e012c81d5127fe6890af6cb81d66a1b768fbe9d3e642898363b39bc","shortlist":"95777acb53d562c552bc5825ddd06bc26919172c065ff5f93d3bbb8467f5d743","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"candidate_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/candidates/manifest.json","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","proxy_schedule":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/schedules/proxy-071-candidate-bc3aed01c13dd69fa821/manifest.json","shortlist":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":["experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"stage":"train_proxy"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..13da5253f5fdc268ef924938b7be064d6deca35d --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT @@ -0,0 +1 @@ +07f877930df296e9e8c63789ad01ec604e875bb1e3dee0748ae9a600bb35c5a2 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..55e87a305d245b435e454a9f1ecb03dbf4eb4d1b Binary files /dev/null and b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata differ diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7ee635e456d701672d154bf1ebd86a734f6b7eba --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bd220e157ade73b70022ecadb2717a972b6936f98d5f978a32f4bb63357f02c +size 254224 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..fee2ac1316750f2d60d2dfd48d153769db8a306e --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/manifest.json @@ -0,0 +1 @@ +{"checkpoint_id":"shadow-bank-proxy","component_names":["shadow_bank"],"content_hash":"07f877930df296e9e8c63789ad01ec604e875bb1e3dee0748ae9a600bb35c5a2","files":{"dcp/.metadata":"d2b3ec769c754bd6c1e1aff78e473fd9767fd8d1aa5cfc88ca6a644a3888b780","dcp/__0_0.distcp":"2bd220e157ade73b70022ecadb2717a972b6936f98d5f978a32f4bb63357f02c","state.pt":"84018ab5bd9bcc03a4ec66ff750e316032f88fb6a58c5b0d742dab0a60e90939"},"identity":{"checkpoint_layout":"dcp_with_local_payload_v1","config_hash":"a26691553c149c42fcc9ee89dcd098e6ec49d11ee75127e96c14ddfda9344dc2","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":"dc0cb6347196f976c34e2e0f5aacca353b9e55582fa5240b662f2f57a87ca167","mesh":{"data_parallel":1,"expert_mapping_version":"contiguous_global_expert_v1","expert_parallel":1,"world_size":1},"model_compatibility":{"payload":{"interface_structure":{"formula_version":"directed_cross_gate_v1","max_interfaces_per_token":1,"norm_epsilon":1e-05,"null_logit_mode":"per_layer_fp32_scalar_v1","pair_router_has_null":true,"pair_router_temperature":1.0,"rank":16,"slot_capacity":1},"model":{"depth":12,"expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_width":384,"hidden_size":384,"in_channels":4,"learn_sigma":false,"moe_layer_pattern":[0,1,2,3,4,5,6,7,8,9,10,11],"num_classes":100,"num_heads":6,"out_channels":4,"patch_size":4,"variant":"tiny"},"router":{"aux_loss_weight":0.0,"correction_bias_clip":0.1,"correction_bias_eta":0.001,"correction_bias_update_interval":10,"correction_bias_warmup_steps":0,"gate_init_std":0.01,"group_score_top_n":2,"load_balancer":"correction_bias","num_experts":4,"num_groups":1,"route_scale":1.0,"shared_expert":true,"top_k":2,"topk_group":1},"runtime_layout":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","expert_backend":"transformer_engine_grouped_linear_bf16_v1","expert_mapping_version":"contiguous_global_expert_v1","precision":"bf16","profile_id":"h100_sm90_te_bf16_nv24_10_v1"}},"signature":"a424098031c3112246695b58cb832e27fb28305df596984aabfd6650aef19698","version":"model_compatibility_signature_v1"},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","role_spec_hash":"bc27fc8428b29ff36a023165e6dd27446e64d1ae7a4a485a7deb6c5444b7676c","schema_version":"1.0.0","shadow_commit_hash":"930a1d4a89b6230735bf759ff11ab0f3e748296b423304fb3100ed021a5fd500","split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"train_state":{"image_exposures":25528,"latent_tokens":0,"microstep":0,"optimizer_step":100,"phase":"base","phase_step":0,"sampler_cursor":null},"world_size":1} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..220c157ed9d430eca0a1966d907e57e6155363d5 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/proxy/checkpoints/shadow-bank-proxy/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84018ab5bd9bcc03a4ec66ff750e316032f88fb6a58c5b0d742dab0a60e90939 +size 3165058 diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_manifest.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..546f688c15a00029325294c91f7fbca0631ad843 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"hash":"eafd077e4dc8a979eeda1999deab239e538d4f024df093fe243ce4589e1da818","name":"proxy_cache_manifest","parent_hash":null,"path":"proxy/cache/manifest.json","schema_version":"1.0.0"},{"hash":"22e26bd6fbe974d69a07f8803e472a6779b55e2bde3868daefeeb713b1dd0896","name":"proxy_evidence","parent_hash":null,"path":"proxy/evidence/manifest.json","schema_version":"1.0.0"},{"hash":"a58cb43ce664aa02f62cfe7339a4ff1ba7985973f5c0e6b846271d05ed677912","name":"proxy_windows","parent_hash":null,"path":"proxy/proxy_windows.json","schema_version":"1.0.0"},{"hash":"92709f9102a5b1303935c3788ffd1ccfe36d8e6b9cdacc2507336e82b893632d","name":"shadow_bank_checkpoint","parent_hash":null,"path":"proxy/checkpoints/shadow-bank-proxy/manifest.json","schema_version":"1.0.0"},{"hash":"41fb10306dcddeb5d4ea7d920ef59333c01cd09f338fccc34bc5aa893a516879","name":"shadow_bank_receipt","parent_hash":null,"path":"proxy/bank_receipt/manifest.json","schema_version":"1.0.0"},{"hash":"bfd9367565f00e8338f2f4a4ed34f074ad9b2d551474e39912761ad8d631e646","name":"launch","parent_hash":null,"path":"launch.json","schema_version":"1.0.0"}],"code":{"contract_version":"contracts_v1","dirty":true,"dirty_patch_hash":"893274bcdbe1420557cf2695ecb0c9806c799efb46bf1b7ce56699a698f636b2","git_commit":"3abed5e6d10894ea1551b5d2b97afa194db72ff3"},"config_hash":"4c3ad4fbd6c0b21c2c19745e27516f0e4a557b16c7666aaa109a2b525da5b8be","content_hash":"43645a1d68826055c7eca3a28a9338b348b6f37d8615fe4a398bfaa5d6d59394","created_at":"2026-08-03T19:29:31.025984+00:00","data_manifest_hash":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","edge_manifest_hash":null,"environment":{"cuda":"12.6","gpus":[{"compute_capability":"9.0","driver":"590.48.01","memory_bytes":85019328512,"name":"NVIDIA H100 80GB HBM3"}],"nccl":"2.22.3","python":"3.10.12","pytorch":"2.5.0a0+e000cf0ad9.nv24.10"},"flops":{"estimated_active":0.0,"measured":null,"method_version":"not_measured_preflight_v1"},"parameters":{"active":0,"logical":0},"parent_checkpoint_hash":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","run_id":"E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e","runtime":{"canonical_weight_alignment_version":"expert_swiglu_gate_up_down_v1","compute_dtype":"bfloat16","cuda_version":"12.6","driver_version":"590.48.01","expert_backend":"transformer_engine_grouped_linear_bf16_v1","profile_hash":"7930764e0384323c317c49db511e455f3392b9e7b73cac7be06927b515007802","profile_id":"h100_sm90_te_bf16_nv24_10_v1","python_version":"3.10.12","pytorch_build":"2.5.0a0+e000cf0ad9.nv24.10","sm":"9.0","transformer_engine_version":"1.11.0+4df8488"},"schema_version":"2.0.0","seeds":{"arm_scheduling":1010,"augmentation":1006,"candidate_audit":6006,"capture_sampling":1007,"cfg":1005,"data_order":1001,"diffusion":1003,"generation":1011,"global_seed":1701,"posterior":1002,"shadow_init":1009,"timestep":1004},"split_manifest_hash":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716","stage":"train_proxy","vae_hash":null,"world_mesh":{"data_parallel":1,"expert_mapping_version":"rank_major_expert_contiguous_v1","expert_parallel":1,"layout_hash":"bbb880479a3d8f7e12e020ecdcb9c55f1ae88d077cbc01b940f1e9c5277207db","world_size":1}} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_status.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_status.json new file mode 100644 index 0000000000000000000000000000000000000000..ee317fccaece4d0b7d3e5130bde6301483e4a32a --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt2/runs/E2-fullv2-base1701-split1001-twin_proxy-p071-3abed5e/run_status.json @@ -0,0 +1 @@ +{"manifest_hash":"43645a1d68826055c7eca3a28a9338b348b6f37d8615fe4a398bfaa5d6d59394","reason":null,"status":"COMMITTED","updated_at":"2026-08-03T19:29:32.019450+00:00"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/launch.json b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..fec9e909c809f0af79ebd637200cd70cf7a7a745 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/launch.json @@ -0,0 +1 @@ +{"adapter":"__main__:proxy_stage_service","argv":["--run-id","E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e","--runs-root","/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs","--config-dir","/workspace/gsh/tmp/codex_tel_work/Schatten_MoE/Schatten_MoE-main/configs","--input","checkpoint=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","--input","data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","--input","split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","--input","shortlist=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","--input","candidate_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/candidates/manifest.json","--input","proxy_schedule=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/schedules/proxy-layer-00/manifest.json","experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"input_hashes":{"candidate_manifest":"d563804591bfa5acb8792eca7150a6388e5958883eac3f6a846b33fcdefcd37a","checkpoint":"919f7a9846ea98d759ecb06343a8eb8da4983480c8a07b569ee231722bee12fb","data_manifest":"4ec206f8c63b542a5210ffa77d3cfa122379bf97b7e87437a2ff34912f59a7cc","proxy_schedule":"49f041b3f2bade45e60b70629c58d2cce4351e69b3f57a60620067c2968f7598","shortlist":"95777acb53d562c552bc5825ddd06bc26919172c065ff5f93d3bbb8467f5d743","split_manifest":"f2d99809dd21966190a823e684aaadda994281f5ccc2ba348db027fa6b43d716"},"inputs":{"candidate_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/candidates/manifest.json","checkpoint":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-basev6-5m-f49af44/base/checkpoints/step-00019532","data_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","proxy_schedule":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/schedules/proxy-layer-00/manifest.json","shortlist":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/runs/E2-sc40-base1701-split1001-observerv2a2-basev6-f49af44/observe/shortlist/manifest.json","split_manifest":"/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json"},"overrides":["experiment=proxy","model=tiny_e2","router=tiny_e2_repair_v6","optimizer=adamw_e2_router_stable_v1","interface=tiny_e2","runtime=h100_te_bf16_nv24_10","distributed=single","output_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs","data.dataset_root=/tmp/gsh/datasets/imagenet1k","data.latent_root=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data","data.data_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/preprocess/data/manifest.json","data.split_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/data/splits/manifest.json","data.vae_revision=31f26fdeee1355a5c34592e401dd41e45d25a493","seeds.global_seed=1701","seeds.data_order=1001","interface.rho=0.03","experiment.decision_policy_hash=b7d6663a2552dfa9d98c4ce18673b6123f19414398265db44e95f45e400a91a3","experiment.calibration_manifest=/workspace/gsh/tmp/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/calibration/e2-sc40-pilot-basev6-v1/manifest.json"],"stage":"train_proxy"} \ No newline at end of file diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT new file mode 100644 index 0000000000000000000000000000000000000000..340e1aecebbde93284f16eedb0675566888ab883 --- /dev/null +++ b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/COMMIT @@ -0,0 +1 @@ +a5e7feffc9d9acb7e7181ad5dd3b0fafc3451ca42a801cf1e3325ec75abc12df diff --git a/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ad0a18f080f41a2e996ee9a18e5898414f8fa0d6 Binary files /dev/null and b/artifacts/Schatten_MoE-singlecard-experiments/E2-SC40-imagenet100-tiny-main-f49af44/continuation/e2-full-v2-attempt5-mps-optimized/runs/E2-fullv2-base1701-split1001-twin_proxy-layer00-3abed5e/proxy/checkpoints/shadow-bank-proxy/dcp/.metadata differ